Slapping together some test code to see if prefill can detect corrupt…

… chunks

SteamPrefill/CliCommands/Benchmark/BenchmarkRunCommand.cs

@@ -87,12 +87,19 @@ private async Task InitializeAsync()
                 _cdnPool = new CdnPool(_ansiConsole, benchmarkWorkload.CdnServerList);
                 _totalDownloadSize = benchmarkWorkload.TotalDownloadSize;
 
-                // Randomizing request order, to simulate a more "realistic" workload similar to Lan Party traffic.
-                // We also want to avoid sequential reads, as they might be getting cached by the server's ram
-                _allRequests = benchmarkWorkload.AllQueuedRequests;
-                _allRequests.Shuffle();
+
+                // Taking a subset of the requests
+                //var subset = benchmarkWorkload.AllQueuedRequests.Take(500).ToList();
+                _allRequests = benchmarkWorkload.AllQueuedRequests.Take(25000).ToList();
+
+                //for (int i = 0; i < 10; i++)
+                //{
+                //    _allRequests.AddRange(subset);
+                //}
+                _totalDownloadSize = ByteSize.FromBytes(_allRequests.Sum(e => e.CompressedLength));
             });
 
+
             _downloadHandler = new DownloadHandler(_ansiConsole, _cdnPool);
             await _downloadHandler.InitializeAsync();
 

SteamPrefill/CliCommands/ValidateCommand.cs

@@ -0,0 +1,91 @@
+// ReSharper disable MemberCanBePrivate.Global - Properties used as parameters can't be private with CliFx, otherwise they won't work.
+namespace SteamPrefill.CliCommands
+{
+    //TODO description
+    //TODO write documentation
+    //TODO split up internal logic so that the prefill and validation logic don't overlap too much
+    [UsedImplicitly]
+    [Command("validate", Description = "TODO")]
+    public class ValidateCommand : ICommand
+    {
+        [CommandOption("all", Description = "Validates all currently owned games", Converter = typeof(NullableBoolConverter))]
+        public bool? DownloadAllOwnedGames { get; init; }
+
+
+        [CommandOption("force", 'f',
+            Description = "Forces the validation to always run, overrides the default behavior of only validating if a newer version is available.",
+            Converter = typeof(NullableBoolConverter))]
+        public bool? Force { get; init; }
+
+        [CommandOption("os", Description = "Specifies which operating system(s) games should be downloaded for.  Can be windows/linux/macos",
+            Converter = typeof(OperatingSystemConverter), Validators = new[] { typeof(OperatingSystemValidator) })]
+        public IReadOnlyList<OperatingSystem> OperatingSystems { get; init; } = new List<OperatingSystem> { OperatingSystem.Windows };
+
+        [CommandOption("verbose", Description = "Produces more detailed log output. Will output logs for games are already up to date.", Converter = typeof(NullableBoolConverter))]
+        public bool? Verbose
+        {
+            get => AppConfig.VerboseLogs;
+            init => AppConfig.VerboseLogs = value ?? default(bool);
+        }
+
+        [CommandOption("unit",
+            Description = "Specifies which unit to use to display download speed.  Can be either bits/bytes.",
+            Converter = typeof(TransferSpeedUnitConverter))]
+        public TransferSpeedUnit TransferSpeedUnit { get; init; } = TransferSpeedUnit.Bits;
+
+        [CommandOption("no-ansi",
+            Description = "Application output will be in plain text.  " +
+                          "Should only be used if terminal does not support Ansi Escape sequences, or when redirecting output to a file.",
+            Converter = typeof(NullableBoolConverter))]
+        public bool? NoAnsiEscapeSequences { get; init; }
+
+        private IAnsiConsole _ansiConsole;
+
+        public async ValueTask ExecuteAsync(IConsole console)
+        {
+            _ansiConsole = console.CreateAnsiConsole();
+            // Property must be set to false in order to disable ansi escape sequences
+            _ansiConsole.Profile.Capabilities.Ansi = !NoAnsiEscapeSequences ?? true;
+
+            await UpdateChecker.CheckForUpdatesAsync(typeof(Program), "tpill90/steam-lancache-prefill", AppConfig.CacheDir);
+
+            var downloadArgs = new DownloadArguments
+            {
+                Force = Force ?? default(bool),
+                NoCache = default(bool),
+                TransferSpeedUnit = TransferSpeedUnit,
+                OperatingSystems = OperatingSystems.ToList()
+            };
+
+            using var steamManager = new SteamManager(_ansiConsole, downloadArgs);
+            ValidateUserHasSelectedApps(steamManager);
+
+            try
+            {
+                await steamManager.InitializeAsync();
+                await steamManager.DownloadMultipleAppsAsync(DownloadAllOwnedGames ?? default(bool), default(bool), 0);
+            }
+            finally
+            {
+                steamManager.Shutdown();
+            }
+        }
+
+        // Validates that the user has selected at least 1 app
+        private void ValidateUserHasSelectedApps(SteamManager steamManager)
+        {
+            var userSelectedApps = steamManager.LoadPreviouslySelectedApps();
+
+            if ((DownloadAllOwnedGames ?? default(bool)) || userSelectedApps.Any())
+            {
+                return;
+            }
+
+            _ansiConsole.MarkupLine(Red("No apps have been selected for prefill! At least 1 app is required!"));
+            _ansiConsole.MarkupLine(Red($"Use the {Cyan("select-apps")} command to interactively choose which apps to prefill. "));
+            _ansiConsole.MarkupLine("");
+            _ansiConsole.Markup(Red($"Alternatively, the flag {LightYellow("--all")} can be specified to prefill all owned apps"));
+            throw new CommandException(".", 1, true);
+        }
+    }
+}

SteamPrefill/Handlers/DepotHandler.cs

@@ -137,6 +137,9 @@ public async Task<List<QueuedRequest>> BuildChunkDownloadQueueAsync(List<DepotIn
             var chunkQueue = new List<QueuedRequest>();
             foreach (var depotManifest in depotManifests)
             {
+                var depot = depots.First(e => e.DepotId == depotManifest.DepotId);
+                var depotKey = await _steam3Session.RequestDepotKeyAsync(depotManifest.DepotId, depot.ContainingAppId);
+
                 // A depot will contain multiple files, that are broken up into 1MB chunks
                 var dedupedChunks = depotManifest.Files
                                                  .SelectMany(e => e.Chunks)
@@ -146,7 +149,7 @@ public async Task<List<QueuedRequest>> BuildChunkDownloadQueueAsync(List<DepotIn
 
                 foreach (ChunkData chunk in dedupedChunks)
                 {
-                    chunkQueue.Add(new QueuedRequest(depotManifest, chunk));
+                    chunkQueue.Add(new QueuedRequest(depotManifest, chunk, depotKey));
                 }
             }
             return chunkQueue;

SteamPrefill/Handlers/DownloadHandler.cs

@@ -1,4 +1,7 @@
-namespace SteamPrefill.Handlers
+using Microsoft.IO;
+using System;
+
+namespace SteamPrefill.Handlers
 {
     public sealed class DownloadHandler : IDisposable
     {
@@ -43,6 +46,7 @@ public async Task<bool> DownloadQueuedChunksAsync(List<QueuedRequest> queuedRequ
             await _ansiConsole.CreateSpectreProgress(downloadArgs.TransferSpeedUnit).StartAsync(async ctx =>
             {
                 // Run the initial download
+                //TODO needs to switch to saying Validating instead of Downloading if validation is running
                 failedRequests = await AttemptDownloadAsync(ctx, "Downloading..", queuedRequests, downloadArgs);
 
                 // Handle any failed requests
@@ -65,6 +69,9 @@ await _ansiConsole.CreateSpectreProgress(downloadArgs.TransferSpeedUnit).StartAs
         }
 
         //TODO I don't like the number of parameters here, should maybe rethink the way this is written.
+        //TODO move somewhere.  I wonder how this affects performance
+        public static readonly RecyclableMemoryStreamManager MemoryStreamManager = new RecyclableMemoryStreamManager();
+
         /// <summary>
         /// Attempts to download the specified requests.  Returns a list of any requests that have failed for any reason.
         /// </summary>
@@ -83,35 +90,56 @@ public async Task<ConcurrentBag<QueuedRequest>> AttemptDownloadAsync(ProgressCon
             {
                 try
                 {
+                    using var cts = new CancellationTokenSource();
                     var url = $"http://{_lancacheAddress}/depot/{request.DepotId}/chunk/{request.ChunkId}";
                     if (forceRecache)
                     {
                         url += "?nocache=1";
                     }
                     using var requestMessage = new HttpRequestMessage(HttpMethod.Get, url);
                     requestMessage.Headers.Host = cdnServer.Host;
+                    var response = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
+                    await using Stream responseStream = await response.Content.ReadAsStreamAsync(cts.Token);
 
-                    using var cts = new CancellationTokenSource();
-                    using var response = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
-                    using Stream responseStream = await response.Content.ReadAsStreamAsync(cts.Token);
-                    response.EnsureSuccessStatusCode();
+                    //TODO Copy to another stream for some reason?
+                    var outputStream = MemoryStreamManager.GetStream();
+                    await responseStream.CopyToAsync(outputStream, cts.Token);
+
+                    // Decrypt first
+                    byte[] encryptedChunkData = outputStream.ToArray();
+
+                    // TODO for some reason not getting the depot key here.  MW2 beta is failing for example
+                    if (request.DepotKey == null)
+                    {
+                        return;
+                    }
+                    byte[] decrypted = CryptoHelper.SymmetricDecrypt(encryptedChunkData, request.DepotKey);
+                    // TODO This is a large amount of the performance hit
+                    byte[] decompressed = DecompressTheShit(decrypted);
+
+                    byte[] computedHash = CryptoHelper.AdlerHash(decompressed);
+                    string computedHashString = HexMate.Convert.ToHexString(computedHash, HexFormattingOptions.Lowercase);
 
-                    // Don't save the data anywhere, so we don't have to waste time writing it to disk.
-                    var buffer = new byte[4096];
-                    while (await responseStream.ReadAsync(buffer, cts.Token) != 0)
+                    if (computedHashString != request.ExpectedChecksumString)
                     {
+                        throw new ChunkChecksumFailedException($"Request {url} failed CRC check.  Will attempt repair");
                     }
                 }
+                catch (ChunkChecksumFailedException e)
+                {
+                    failedRequests.Add(request);
+                    _ansiConsole.LogMarkupLine(Red(e.Message));
+                    FileLogger.LogExceptionNoStackTrace(e.Message, e);
+                }
                 catch (Exception e)
                 {
+                    _ansiConsole.LogMarkupVerbose(Red($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed : {e.GetType()}"));
                     failedRequests.Add(request);
-                    _ansiConsole.LogMarkupLine(Red($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed : {e.GetType()}"));
                     FileLogger.LogExceptionNoStackTrace($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed", e);
                 }
                 progressTask.Increment(request.CompressedLength);
             });
 
-            //TODO In the scenario where a user still had all requests fail, potentially display a warning that there is an underlying issue
             // Only return the connections for reuse if there were no errors
             if (failedRequests.IsEmpty)
             {
@@ -123,6 +151,17 @@ public async Task<ConcurrentBag<QueuedRequest>> AttemptDownloadAsync(ProgressCon
             return failedRequests;
         }
 
+        private static byte[] DecompressTheShit(byte[] decrypted)
+        {
+            if (decrypted.Length > 1 && decrypted[0] == 'V' && decrypted[1] == 'Z')
+            {
+                // LZMA
+                return VZipUtil.Decompress(decrypted);
+            }
+            // Deflate
+            return ZipUtil.Decompress(decrypted);
+        }
+
         public void Dispose()
         {
             _client?.Dispose();

SteamPrefill/Handlers/Steam/Steam3Session.cs

@@ -356,6 +356,13 @@ private void LicenseListCallback(SteamApps.LicenseListCallback licenseList)
 
         #endregion
 
+        public async Task<byte[]> RequestDepotKeyAsync(uint depotId, uint appid = 0)
+        {
+            var response = await SteamAppsApi.GetDepotDecryptionKey(depotId, appid).ToTask();
+
+            return response.DepotKey;
+        }
+
         public void Dispose()
         {
             CdnClient.Dispose();

SteamPrefill/Models/Exceptions/ChunkChecksumFailedException.cs

@@ -0,0 +1,17 @@
+namespace SteamPrefill.Models.Exceptions
+{
+    public class ChunkChecksumFailedException : Exception
+    {
+        public ChunkChecksumFailedException(string message) : base(message)
+        {
+        }
+
+        public ChunkChecksumFailedException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+
+        public ChunkChecksumFailedException()
+        {
+        }
+    }
+}

SteamPrefill/Models/Manifest.cs

@@ -74,7 +74,8 @@ public FileData(DepotManifest.FileData sourceData) : this()
     public readonly struct ChunkData
     {
         /// <summary>
-        /// SHA-1 hash of the chunk, used as its Id.
+        /// SHA-1 hash of the chunk, used as its Id.  Always 20 bytes, but converted to a string so it can be
+        /// directly used to make a web request.
         /// </summary>
         [ProtoMember(1)]
         public readonly string ChunkId;
@@ -85,10 +86,23 @@ public readonly struct ChunkData
         [ProtoMember(2)]
         public readonly uint CompressedLength;
 
+        /// <summary>
+        /// Adler-32 hash, always 4 bytes
+        /// </summary>
+        [ProtoMember(3)]
+        public readonly byte[] Checksum;
+
+        //TODO remove?
+        [ProtoMember(4)]
+        public readonly string ChecksumString;
+
         public ChunkData(DepotManifest.ChunkData sourceChunk)
         {
             ChunkId = HexMate.Convert.ToHexString(sourceChunk.ChunkID, HexFormattingOptions.Lowercase);
             CompressedLength = sourceChunk.CompressedLength;
+
+            Checksum = sourceChunk.Checksum;
+            ChecksumString = HexMate.Convert.ToHexString(sourceChunk.Checksum, HexFormattingOptions.Lowercase);
         }
 
         public override string ToString()

SteamPrefill/Models/QueuedRequest.cs

@@ -18,11 +18,29 @@ public struct QueuedRequest
         [ProtoMember(3)]
         public readonly uint CompressedLength;
 
-        public QueuedRequest(Manifest depotManifest, ChunkData chunk)
+        /// <summary>
+        /// Adler-32 hash, always 4 bytes
+        /// </summary>
+        [ProtoMember(4)]
+        public readonly byte[] ExpectedChecksum;
+
+        //TODO remove?
+        [ProtoMember(5)]
+        public readonly string ExpectedChecksumString;
+
+        [ProtoMember(6)]
+        public readonly byte[] DepotKey;
+
+        public QueuedRequest(Manifest depotManifest, ChunkData chunk, byte[] depotKey)
         {
             DepotId = depotManifest.DepotId;
             ChunkId = chunk.ChunkId;
             CompressedLength = chunk.CompressedLength;
+
+            ExpectedChecksum = chunk.Checksum;
+            ExpectedChecksumString = chunk.ChecksumString;
+
+            DepotKey = depotKey;
         }
 
         public override string ToString()

SteamPrefill/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "Prefill All": {
       "commandName": "Project",
-      "commandLineArgs": "prefill --all --force --verbose"
+      "commandLineArgs": "prefill --all --verbose"
     },
     "Prefill All - No Download": {
       "commandName": "Project",
@@ -34,11 +34,15 @@
     },
     "Benchmark - Run": {
       "commandName": "Project",
-      "commandLineArgs": "benchmark run"
+      "commandLineArgs": "benchmark run -c 4 -i 2"
     },
     "Clear cache": {
       "commandName": "Project",
-      "commandLineArgs": "clear-cache"
+      "commandLineArgs": "clear-cache -y"
+    },
+    "Validate": {
+      "commandName": "Project",
+      "commandLineArgs": "validate --force --verbose"
     }
   }
 }

SteamPrefill/Settings/AppConfig.cs

@@ -1,5 +1,6 @@
 namespace SteamPrefill.Settings
 {
+    //TODO move to root folder
     public static class AppConfig
     {
         static AppConfig()
@@ -57,7 +58,7 @@ public static bool EnableSteamKitDebugLogs
         /// <summary>
         /// Increment when there is a breaking change made to the files in the cache directory
         /// </summary>
-        private const string CacheDirVersion = "v1";
+        private const string CacheDirVersion = "v2";
 
         /// <summary>
         /// Contains user configuration.  Should not be deleted, doing so will reset the app back to defaults.