Skip to content

Commit

Permalink
Slapping together some test code to see if prefill can detect corrupt…
Browse files Browse the repository at this point in the history
… chunks
  • Loading branch information
tpill90 committed Feb 3, 2024
1 parent 9cbd4d1 commit 1401d8c
Show file tree
Hide file tree
Showing 10 changed files with 206 additions and 30 deletions.
91 changes: 91 additions & 0 deletions SteamPrefill/CliCommands/ValidateCommand.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// ReSharper disable MemberCanBePrivate.Global - Properties used as parameters can't be private with CliFx, otherwise they won't work.
namespace SteamPrefill.CliCommands
{
//TODO description
//TODO write documentation
//TODO split up internal logic so that the prefill and validation logic don't overlap too much
[UsedImplicitly]
[Command("validate", Description = "TODO")]
public class ValidateCommand : ICommand
{
[CommandOption("all", Description = "Validates all currently owned games", Converter = typeof(NullableBoolConverter))]
public bool? DownloadAllOwnedGames { get; init; }


[CommandOption("force", 'f',
Description = "Forces the validation to always run, overrides the default behavior of only validating if a newer version is available.",
Converter = typeof(NullableBoolConverter))]
public bool? Force { get; init; }

[CommandOption("os", Description = "Specifies which operating system(s) games should be downloaded for. Can be windows/linux/macos",
Converter = typeof(OperatingSystemConverter), Validators = new[] { typeof(OperatingSystemValidator) })]
public IReadOnlyList<OperatingSystem> OperatingSystems { get; init; } = new List<OperatingSystem> { OperatingSystem.Windows };

[CommandOption("verbose", Description = "Produces more detailed log output. Will output logs for games are already up to date.", Converter = typeof(NullableBoolConverter))]
public bool? Verbose
{
get => AppConfig.VerboseLogs;
init => AppConfig.VerboseLogs = value ?? default(bool);
}

[CommandOption("unit",
Description = "Specifies which unit to use to display download speed. Can be either bits/bytes.",
Converter = typeof(TransferSpeedUnitConverter))]
public TransferSpeedUnit TransferSpeedUnit { get; init; } = TransferSpeedUnit.Bits;

[CommandOption("no-ansi",
Description = "Application output will be in plain text. " +
"Should only be used if terminal does not support Ansi Escape sequences, or when redirecting output to a file.",
Converter = typeof(NullableBoolConverter))]
public bool? NoAnsiEscapeSequences { get; init; }

private IAnsiConsole _ansiConsole;

public async ValueTask ExecuteAsync(IConsole console)
{
_ansiConsole = console.CreateAnsiConsole();
// Property must be set to false in order to disable ansi escape sequences
_ansiConsole.Profile.Capabilities.Ansi = !NoAnsiEscapeSequences ?? true;

await UpdateChecker.CheckForUpdatesAsync(typeof(Program), "tpill90/steam-lancache-prefill", AppConfig.CacheDir);

var downloadArgs = new DownloadArguments
{
Force = Force ?? default(bool),
NoCache = default(bool),
TransferSpeedUnit = TransferSpeedUnit,
OperatingSystems = OperatingSystems.ToList()
};

using var steamManager = new SteamManager(_ansiConsole, downloadArgs);
ValidateUserHasSelectedApps(steamManager);

try
{
await steamManager.InitializeAsync();
await steamManager.DownloadMultipleAppsAsync(DownloadAllOwnedGames ?? default(bool), default(bool), 0);
}
finally
{
steamManager.Shutdown();
}
}

// Validates that the user has selected at least 1 app
private void ValidateUserHasSelectedApps(SteamManager steamManager)
{
var userSelectedApps = steamManager.LoadPreviouslySelectedApps();

if ((DownloadAllOwnedGames ?? default(bool)) || userSelectedApps.Any())
{
return;
}

_ansiConsole.MarkupLine(Red("No apps have been selected for prefill! At least 1 app is required!"));
_ansiConsole.MarkupLine(Red($"Use the {Cyan("select-apps")} command to interactively choose which apps to prefill. "));
_ansiConsole.MarkupLine("");
_ansiConsole.Markup(Red($"Alternatively, the flag {LightYellow("--all")} can be specified to prefill all owned apps"));
throw new CommandException(".", 1, true);
}
}
}
5 changes: 4 additions & 1 deletion SteamPrefill/Handlers/DepotHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ public async Task<List<QueuedRequest>> BuildChunkDownloadQueueAsync(List<DepotIn
var chunkQueue = new List<QueuedRequest>();
foreach (var depotManifest in depotManifests)
{
var depot = depots.First(e => e.DepotId == depotManifest.DepotId);
var depotKey = await _steam3Session.RequestDepotKeyAsync(depotManifest.DepotId, depot.ContainingAppId);

// A depot will contain multiple files, that are broken up into 1MB chunks
var dedupedChunks = depotManifest.Files
.SelectMany(e => e.Chunks)
Expand All @@ -146,7 +149,7 @@ public async Task<List<QueuedRequest>> BuildChunkDownloadQueueAsync(List<DepotIn

foreach (ChunkData chunk in dedupedChunks)
{
chunkQueue.Add(new QueuedRequest(depotManifest, chunk));
chunkQueue.Add(new QueuedRequest(depotManifest, chunk, depotKey));
}
}
return chunkQueue;
Expand Down
42 changes: 33 additions & 9 deletions SteamPrefill/Handlers/DownloadHandler.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
namespace SteamPrefill.Handlers
using Microsoft.IO;

namespace SteamPrefill.Handlers
{
public sealed class DownloadHandler : IDisposable
{
Expand Down Expand Up @@ -43,6 +45,7 @@ public async Task<bool> DownloadQueuedChunksAsync(List<QueuedRequest> queuedRequ
await _ansiConsole.CreateSpectreProgress(downloadArgs.TransferSpeedUnit).StartAsync(async ctx =>
{
// Run the initial download
//TODO needs to switch to saying Validating instead of Downloading if validation is running
failedRequests = await AttemptDownloadAsync(ctx, "Downloading..", queuedRequests, downloadArgs);
// Handle any failed requests
Expand All @@ -65,6 +68,9 @@ await _ansiConsole.CreateSpectreProgress(downloadArgs.TransferSpeedUnit).StartAs
}

//TODO I don't like the number of parameters here, should maybe rethink the way this is written.
//TODO move somewhere. I wonder how this affects performance
public static readonly RecyclableMemoryStreamManager MemoryStreamManager = new RecyclableMemoryStreamManager();

/// <summary>
/// Attempts to download the specified requests. Returns a list of any requests that have failed for any reason.
/// </summary>
Expand All @@ -83,6 +89,7 @@ public async Task<ConcurrentBag<QueuedRequest>> AttemptDownloadAsync(ProgressCon
{
try
{
using var cts = new CancellationTokenSource();
var url = $"http://{_lancacheAddress}/depot/{request.DepotId}/chunk/{request.ChunkId}";
if (forceRecache)
{
Expand All @@ -91,21 +98,38 @@ public async Task<ConcurrentBag<QueuedRequest>> AttemptDownloadAsync(ProgressCon
using var requestMessage = new HttpRequestMessage(HttpMethod.Get, url);
requestMessage.Headers.Host = cdnServer.Host;
using var cts = new CancellationTokenSource();
using var response = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
using Stream responseStream = await response.Content.ReadAsStreamAsync(cts.Token);
response.EnsureSuccessStatusCode();
var response = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
await using Stream responseStream = await response.Content.ReadAsStreamAsync(cts.Token);
var outputStream = MemoryStreamManager.GetStream();
await responseStream.CopyToAsync(outputStream, cts.Token);
// Decrypt first
var array = outputStream.ToArray();
byte[] processedData = CryptoHelper.SymmetricDecrypt(array, request.DepotKey);
if (processedData.Length > 1 && processedData[0] == 'V' && processedData[1] == 'Z')
{
processedData = VZipUtil.Decompress(processedData);
}
else
{
processedData = ZipUtil.Decompress(processedData);
}
var computedHash = CryptoHelper.AdlerHash(processedData);
var computedHashString = HexMate.Convert.ToHexString(computedHash, HexFormattingOptions.Lowercase);
var expectedHash = request.ChecksumString;
// Don't save the data anywhere, so we don't have to waste time writing it to disk.
var buffer = new byte[4096];
while (await responseStream.ReadAsync(buffer, cts.Token) != 0)
if (computedHashString != expectedHash)
{
_ansiConsole.LogMarkupLine(Red($"Request {url} is corrupted"));
}
}
catch (Exception e)
{
_ansiConsole.LogMarkupVerbose(Red($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed : {e.GetType()}"));
failedRequests.Add(request);
_ansiConsole.LogMarkupLine(Red($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed : {e.GetType()}"));
FileLogger.LogExceptionNoStackTrace($"Request /depot/{request.DepotId}/chunk/{request.ChunkId} failed", e);
}
progressTask.Increment(request.CompressedLength);
Expand Down
7 changes: 7 additions & 0 deletions SteamPrefill/Handlers/Steam/Steam3Session.cs
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,13 @@ private void LicenseListCallback(SteamApps.LicenseListCallback licenseList)

#endregion

public async Task<byte[]> RequestDepotKeyAsync(uint depotId, uint appid = 0)
{
var response = await SteamAppsApi.GetDepotDecryptionKey(depotId, appid).ToTask();

return response.DepotKey;
}

public void Dispose()
{
CdnClient.Dispose();
Expand Down
16 changes: 15 additions & 1 deletion SteamPrefill/Models/Manifest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ public FileData(DepotManifest.FileData sourceData) : this()
public readonly struct ChunkData
{
/// <summary>
/// SHA-1 hash of the chunk, used as its Id.
/// SHA-1 hash of the chunk, used as its Id. Always 20 bytes, but converted to a string so it can be
/// directly used to make a web request.
/// </summary>
[ProtoMember(1)]
public readonly string ChunkId;
Expand All @@ -85,10 +86,23 @@ public readonly struct ChunkData
[ProtoMember(2)]
public readonly uint CompressedLength;

/// <summary>
/// Adler-32 hash, always 4 bytes
/// </summary>
[ProtoMember(3)]
public readonly byte[] Checksum;

//TODO remove?
[ProtoMember(4)]
public readonly string ChecksumString;

public ChunkData(DepotManifest.ChunkData sourceChunk)
{
ChunkId = HexMate.Convert.ToHexString(sourceChunk.ChunkID, HexFormattingOptions.Lowercase);
CompressedLength = sourceChunk.CompressedLength;

Checksum = sourceChunk.Checksum;
ChecksumString = HexMate.Convert.ToHexString(sourceChunk.Checksum, HexFormattingOptions.Lowercase);
}

public override string ToString()
Expand Down
20 changes: 19 additions & 1 deletion SteamPrefill/Models/QueuedRequest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,29 @@ public struct QueuedRequest
[ProtoMember(3)]
public readonly uint CompressedLength;

public QueuedRequest(Manifest depotManifest, ChunkData chunk)
/// <summary>
/// Adler-32 hash, always 4 bytes
/// </summary>
[ProtoMember(4)]
public readonly byte[] Checksum;

//TODO remove?
[ProtoMember(5)]
public readonly string ChecksumString;

[ProtoMember(6)]
public readonly byte[] DepotKey;

public QueuedRequest(Manifest depotManifest, ChunkData chunk, byte[] depotKey)
{
DepotId = depotManifest.DepotId;
ChunkId = chunk.ChunkId;
CompressedLength = chunk.CompressedLength;

Checksum = chunk.Checksum;
ChecksumString = chunk.ChecksumString;

DepotKey = depotKey;
}

public override string ToString()
Expand Down
6 changes: 5 additions & 1 deletion SteamPrefill/Properties/launchSettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,11 @@
},
"Clear cache": {
"commandName": "Project",
"commandLineArgs": "clear-cache"
"commandLineArgs": "clear-cache -y"
},
"Validate": {
"commandName": "Project",
"commandLineArgs": "validate --force --verbose"
}
}
}
3 changes: 2 additions & 1 deletion SteamPrefill/Settings/AppConfig.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
namespace SteamPrefill.Settings
{
//TODO move to root folder
public static class AppConfig
{
static AppConfig()
Expand Down Expand Up @@ -57,7 +58,7 @@ public static bool EnableSteamKitDebugLogs
/// <summary>
/// Increment when there is a breaking change made to the files in the cache directory
/// </summary>
private const string CacheDirVersion = "v1";
private const string CacheDirVersion = "v2";

/// <summary>
/// Contains user configuration. Should not be deleted, doing so will reset the app back to defaults.
Expand Down
46 changes: 30 additions & 16 deletions SteamPrefill/SteamPrefill.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,27 @@

<!-- Publish Settings -->
<PropertyGroup>
<PublishTrimmed>true</PublishTrimmed>
<EnableTrimAnalyzer>true</EnableTrimAnalyzer>
<PublishTrimmed>true</PublishTrimmed>
<EnableTrimAnalyzer>true</EnableTrimAnalyzer>
<TrimmerSingleWarn>false</TrimmerSingleWarn>
<!-- Reverts back to the default trim mode used by dotnet 6, otherwise CliFx breaks without reflection -->
<TrimMode>partial</TrimMode>
<TrimMode>partial</TrimMode>

<PublishReadyToRun>true</PublishReadyToRun>
<PublishSingleFile>true</PublishSingleFile>

<!-- Required to be enabled in order to run this application on Ubuntu Docker images. -->
<InvariantGlobalization>true</InvariantGlobalization>

<!-- Removes the git commit hash being appended to the version number when publishing.. Ex: v2.3.0+5afde434cfe8472ba36138c4912e7aa08a7a22d0 -->
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<DebugType>none</DebugType>
<DebugSymbols>false</DebugSymbols>
</PropertyGroup>

<InvariantGlobalization>true</InvariantGlobalization>

<!-- Removes the git commit hash being appended to the version number when publishing.. Ex: v2.3.0+5afde434cfe8472ba36138c4912e7aa08a7a22d0 -->
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<DebugType>none</DebugType>
<DebugSymbols>false</DebugSymbols>
</PropertyGroup>

<!--TODO add this to the other projects-->
<PropertyGroup>
<!-- Removes the full file system path from exception stack traces, only shows the file names now -->
Expand Down Expand Up @@ -67,14 +67,27 @@
<PackageReference Include="AutoMapper" Version="11.0.1" />
<PackageReference Include="HexMate" Version="0.0.3" />
<PackageReference Include="Intellenum" Version="1.0.0-beta.3" />
<PackageReference Include="protobuf-net" Version="3.2.16" />
<PackageReference Include="SteamKit2" Version="2.5.0-Beta.1" />
<PackageReference Include="protobuf-net" Version="3.2.26" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.34.0" />
<PackageReference Include="ThisAssembly.AssemblyInfo" Version="1.2.14">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>

<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.2.1" />
<PackageReference Include="SharpCompress" Version="0.32.2" />
<PackageReference Include="Terminal.Gui" Version="1.7.2" />
<PackageReference Include="Wcwidth" Version="1.0.0" />
<PackageReference Include="ZString" Version="2.4.4" />

<!-- Custom SteamKit2 build, some internal methods have been made public so that we can use them here -->
<PackageReference Include="System.IO.Hashing" Version="8.0.0" />
<Reference Include="SteamKit2">
<HintPath>..\lib\SteamKit2.dll</HintPath>
</Reference>

<!--<ProjectReference Include="..\..\ExternalLibraries\SteamKit\SteamKit2\SteamKit2\SteamKit2.csproj" />-->

<!-- Custom CliFx build, allows for top level exception handling in application code, instead of CliFx swallowing all exceptions -->
<Reference Include="CliFx">
<HintPath>..\LancachePrefill.Common\lib\CliFx.dll</HintPath>
Expand Down Expand Up @@ -116,6 +129,7 @@
</ItemGroup>

<ItemGroup>

<ProjectReference Include="..\LancachePrefill.Common\dotnet\LancachePrefill.Common.csproj" />
</ItemGroup>

Expand Down
Binary file added lib/SteamKit2.dll
Binary file not shown.

0 comments on commit 1401d8c

Please sign in to comment.