diff --git a/SteamKit2/SteamKit2/Steam/CDN/Client.cs b/SteamKit2/SteamKit2/Steam/CDN/Client.cs
index 55f73285f..dfb774cf7 100644
--- a/SteamKit2/SteamKit2/Steam/CDN/Client.cs
+++ b/SteamKit2/SteamKit2/Steam/CDN/Client.cs
@@ -4,7 +4,9 @@
*/
using System;
+using System.Buffers;
using System.IO;
+using System.IO.Compression;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
@@ -58,7 +60,7 @@ public void Dispose()
/// The content server to connect to.
///
/// The depot decryption key for the depot that will be downloaded.
- /// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
+ /// This is used for decrypting filenames (if needed) in depot manifests.
///
/// Optional content server marked as UseAsProxy which transforms the request.
/// A instance that contains information about the files present within a depot.
@@ -81,11 +83,65 @@ public async Task DownloadManifestAsync( uint depotId, ulong mani
url = $"depot/{depotId}/manifest/{manifestId}/{MANIFEST_VERSION}";
}
- var manifestData = await DoRawCommandAsync( server, url, proxyServer ).ConfigureAwait( false );
+ using var request = new HttpRequestMessage( HttpMethod.Get, BuildCommand( server, url, proxyServer ) );
- manifestData = ZipUtil.Decompress( manifestData );
+ using var cts = new CancellationTokenSource();
+ cts.CancelAfter( RequestTimeout );
+
+ DepotManifest depotManifest;
+
+ try
+ {
+ using var response = await httpClient.SendAsync( request, HttpCompletionOption.ResponseHeadersRead, cts.Token ).ConfigureAwait( false );
+
+ if ( !response.IsSuccessStatusCode )
+ {
+ throw new SteamKitWebRequestException( $"Response status code does not indicate success: {response.StatusCode:D} ({response.ReasonPhrase}).", response );
+ }
+
+ if ( !response.Content.Headers.ContentLength.HasValue )
+ {
+ throw new SteamKitWebRequestException( "Response does not have Content-Length", response );
+ }
+
+ cts.CancelAfter( ResponseBodyTimeout );
+
+ var contentLength = ( int )response.Content.Headers.ContentLength;
+ var buffer = ArrayPool.Shared.Rent( contentLength );
- var depotManifest = new DepotManifest( manifestData );
+ try
+ {
+ using var ms = new MemoryStream( buffer, 0, contentLength );
+
+ // Stream the http response into the rented buffer
+ await response.Content.CopyToAsync( ms, cts.Token );
+
+ if ( ms.Position != contentLength )
+ {
+ throw new InvalidDataException( $"Length mismatch after downloading depot manifest! (was {ms.Position}, but should be {contentLength})" );
+ }
+
+ ms.Position = 0;
+
+ // Decompress the zipped manifest data
+ using var zip = new ZipArchive( ms );
+ var entries = zip.Entries;
+
+ DebugLog.Assert( entries.Count == 1, nameof( CDN ), "Expected the zip to contain only one file" );
+
+ using var zipEntryStream = entries[ 0 ].Open();
+ depotManifest = DepotManifest.Deserialize( zipEntryStream );
+ }
+ finally
+ {
+ ArrayPool.Shared.Return( buffer );
+ }
+ }
+ catch ( Exception ex )
+ {
+ DebugLog.WriteLine( nameof( CDN ), $"Failed to download manifest {request.RequestUri}: {ex.Message}" );
+ throw;
+ }
if ( depotKey != null )
{
@@ -108,53 +164,51 @@ public async Task DownloadManifestAsync( uint depotId, ulong mani
/// A instance that represents the chunk to download.
/// This value should come from a manifest downloaded with .
///
- /// A instance that contains the data for the given chunk.
+ /// The total number of bytes written to .
/// The content server to connect to.
+ ///
+ /// The buffer to receive the chunk data. If is provided, this will be the decompressed buffer.
+ /// Allocate or rent a buffer that is equal or longer than
+ ///
///
/// The depot decryption key for the depot that will be downloaded.
- /// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
+ /// This is used to process the chunk data.
///
/// Optional content server marked as UseAsProxy which transforms the request.
/// chunk's was null.
/// Thrown if the downloaded data does not match the expected length.
/// An network error occurred when performing the request.
/// A network error occurred when performing the request.
- public async Task DownloadDepotChunkAsync( uint depotId, DepotManifest.ChunkData chunk, Server server, byte[]? depotKey = null, Server? proxyServer = null )
+ public async Task DownloadDepotChunkAsync( uint depotId, DepotManifest.ChunkData chunk, Server server, byte[] destination, byte[]? depotKey = null, Server? proxyServer = null )
{
ArgumentNullException.ThrowIfNull( server );
-
ArgumentNullException.ThrowIfNull( chunk );
+ ArgumentNullException.ThrowIfNull( destination );
if ( chunk.ChunkID == null )
{
- throw new ArgumentException( "Chunk must have a ChunkID.", nameof( chunk ) );
+ throw new ArgumentException( $"Chunk must have a {nameof( DepotManifest.ChunkData.ChunkID )}.", nameof( chunk ) );
}
- var chunkID = Utils.EncodeHexString( chunk.ChunkID );
-
- var chunkData = await DoRawCommandAsync( server, string.Format( "depot/{0}/chunk/{1}", depotId, chunkID ), proxyServer ).ConfigureAwait( false );
-
- // assert that lengths match only if the chunk has a length assigned.
- if ( chunk.CompressedLength > 0 && chunkData.Length != chunk.CompressedLength )
+ if ( depotKey == null )
{
- throw new InvalidDataException( $"Length mismatch after downloading depot chunk! (was {chunkData.Length}, but should be {chunk.CompressedLength})" );
+ if ( destination.Length < chunk.CompressedLength )
+ {
+ throw new ArgumentException( $"The destination buffer must be longer than the chunk {nameof( DepotManifest.ChunkData.CompressedLength )} (since no depot key was provided).", nameof( destination ) );
+ }
}
-
- var depotChunk = new DepotChunk( chunk, chunkData );
-
- if ( depotKey != null )
+ else
{
- // if we have the depot key, we can process the chunk immediately
- depotChunk.Process( depotKey );
+ if ( destination.Length < chunk.UncompressedLength )
+ {
+ throw new ArgumentException( $"The destination buffer must be longer than the chunk {nameof( DepotManifest.ChunkData.UncompressedLength )}.", nameof( destination ) );
+ }
}
- return depotChunk;
- }
+ var chunkID = Utils.EncodeHexString( chunk.ChunkID );
+ var url = $"depot/{depotId}/chunk/{chunkID}";
- async Task DoRawCommandAsync( Server server, string command, Server? proxyServer )
- {
- var url = BuildCommand( server, command, proxyServer );
- using var request = new HttpRequestMessage( HttpMethod.Get, url );
+ using var request = new HttpRequestMessage( HttpMethod.Get, BuildCommand( server, url, proxyServer ) );
using var cts = new CancellationTokenSource();
cts.CancelAfter( RequestTimeout );
@@ -168,13 +222,65 @@ async Task DoRawCommandAsync( Server server, string command, Server? pro
throw new SteamKitWebRequestException( $"Response status code does not indicate success: {response.StatusCode:D} ({response.ReasonPhrase}).", response );
}
+ if ( !response.Content.Headers.ContentLength.HasValue )
+ {
+ throw new SteamKitWebRequestException( "Response does not have Content-Length", response );
+ }
+
+ var contentLength = ( int )response.Content.Headers.ContentLength;
+
+ // assert that lengths match only if the chunk has a length assigned.
+ if ( chunk.CompressedLength > 0 && contentLength != chunk.CompressedLength )
+ {
+ throw new InvalidDataException( $"Content-Length mismatch for depot chunk! (was {contentLength}, but should be {chunk.CompressedLength})" );
+ }
+
cts.CancelAfter( ResponseBodyTimeout );
- return await response.Content.ReadAsByteArrayAsync( cts.Token ).ConfigureAwait( false );
+ // If no depot key is provided, stream into the destination buffer without renting
+ if ( depotKey == null )
+ {
+ using var ms = new MemoryStream( destination, 0, contentLength );
+
+ // Stream the http response into the provided destination
+ await response.Content.CopyToAsync( ms, cts.Token );
+
+ if ( ms.Position != contentLength )
+ {
+ throw new InvalidDataException( $"Length mismatch after downloading depot chunk! (was {ms.Position}, but should be {contentLength})" );
+ }
+
+ return contentLength;
+ }
+
+ // We have to stream into a temporary buffer because a decryption will need to be performed
+ var buffer = ArrayPool.Shared.Rent( contentLength );
+
+ try
+ {
+ using var ms = new MemoryStream( buffer, 0, contentLength );
+
+ // Stream the http response into the rented buffer
+ await response.Content.CopyToAsync( ms, cts.Token );
+
+ if ( ms.Position != contentLength )
+ {
+ throw new InvalidDataException( $"Length mismatch after downloading depot chunk! (was {ms.Position}, but should be {contentLength})" );
+ }
+
+ // process the chunk immediately
+ var writtenLength = DepotChunk.Process( chunk, buffer.AsSpan()[ ..contentLength ], destination, depotKey );
+
+ return writtenLength;
+ }
+ finally
+ {
+ ArrayPool.Shared.Return( buffer );
+ }
}
catch ( Exception ex )
{
- DebugLog.WriteLine( nameof( CDN ), "Failed to complete web request to {0}: {1}", url, ex.Message );
+ DebugLog.WriteLine( nameof( CDN ), $"Failed to download a depot chunk {request.RequestUri}: {ex.Message}" );
throw;
}
}
diff --git a/SteamKit2/SteamKit2/Steam/CDN/DepotChunk.cs b/SteamKit2/SteamKit2/Steam/CDN/DepotChunk.cs
index 806e364fb..4dd743691 100644
--- a/SteamKit2/SteamKit2/Steam/CDN/DepotChunk.cs
+++ b/SteamKit2/SteamKit2/Steam/CDN/DepotChunk.cs
@@ -4,84 +4,84 @@
*/
using System;
+using System.Buffers;
using System.IO;
-using System.Linq;
+using System.Security.Cryptography;
namespace SteamKit2.CDN
{
///
- /// Represents a single downloaded chunk from a file in a depot.
+ /// Provides a helper function to decrypt and decompress a single depot chunk.
///
- public sealed class DepotChunk
+ public static class DepotChunk
{
- ///
- /// Gets the depot manifest chunk information associated with this chunk.
- ///
- public DepotManifest.ChunkData ChunkInfo { get; }
-
- ///
- /// Gets a value indicating whether this chunk has been processed. A chunk is processed when the data has been decrypted and decompressed.
- ///
- ///
- /// true if this chunk has been processed; otherwise, false.
- ///
- public bool IsProcessed { get; internal set; }
-
- ///
- /// Gets the underlying data for this chunk.
- ///
- public byte[] Data { get; private set; }
-
- ///
- /// Initializes a new instance of the class.
- ///
- /// The manifest chunk information associated with this chunk.
- /// The underlying data for this chunk.
- public DepotChunk( DepotManifest.ChunkData info, byte[] data )
- {
- ArgumentNullException.ThrowIfNull( info );
-
- ArgumentNullException.ThrowIfNull( data );
-
- ChunkInfo = info;
- Data = data;
- }
-
///
/// Processes the specified depot key by decrypting the data with the given depot encryption key, and then by decompressing the data.
/// If the chunk has already been processed, this function does nothing.
///
+ /// The depot chunk data representing.
+ /// The encrypted chunk data.
+ /// The buffer to receive the decrypted chunk data.
/// The depot decryption key.
- /// Thrown if the processed data does not match the expected checksum given in it's chunk information.
- public void Process( byte[] depotKey )
+ /// Thrown if the processed data does not match the expected checksum given in it's chunk information.
+ public static int Process( DepotManifest.ChunkData info, ReadOnlySpan data, byte[] destination, byte[] depotKey )
{
+ ArgumentNullException.ThrowIfNull( info );
ArgumentNullException.ThrowIfNull( depotKey );
- if ( IsProcessed )
+ if ( destination.Length < info.UncompressedLength )
{
- return;
+ throw new ArgumentException( $"The destination buffer must be longer than the chunk {nameof( DepotManifest.ChunkData.UncompressedLength )}.", nameof( destination ) );
}
- byte[] processedData = CryptoHelper.SymmetricDecrypt( Data, depotKey );
+ DebugLog.Assert( depotKey.Length == 32, nameof( DepotChunk ), $"Tried to decrypt depot chunk with non 32 byte key!" );
+
+ using var aes = Aes.Create();
+ aes.BlockSize = 128;
+ aes.KeySize = 256;
+ aes.Key = depotKey;
+
+ // first 16 bytes of input is the ECB encrypted IV
+ Span iv = stackalloc byte[ 16 ];
+ aes.DecryptEcb( data[ ..iv.Length ], iv, PaddingMode.None );
- if ( processedData.Length > 1 && processedData[ 0 ] == 'V' && processedData[ 1 ] == 'Z' )
+ // With CBC and padding, the decrypted size will always be smaller
+ var buffer = ArrayPool.Shared.Rent( data.Length - iv.Length );
+
+ var writtenDecompressed = 0;
+
+ try
+ {
+ var written = aes.DecryptCbc( data[ iv.Length.. ], iv, buffer, PaddingMode.PKCS7 );
+ var decryptedStream = new MemoryStream( buffer, 0, written );
+
+ if ( buffer.Length > 1 && buffer[ 0 ] == 'V' && buffer[ 1 ] == 'Z' )
+ {
+ writtenDecompressed = VZipUtil.Decompress( decryptedStream, destination, verifyChecksum: false );
+ }
+ else
+ {
+ writtenDecompressed = ZipUtil.Decompress( decryptedStream, destination, verifyChecksum: false );
+ }
+ }
+ finally
{
- processedData = VZipUtil.Decompress( processedData );
+ ArrayPool.Shared.Return( buffer );
}
- else
+
+ if ( info.UncompressedLength != writtenDecompressed )
{
- processedData = ZipUtil.Decompress( processedData );
+ throw new InvalidDataException( $"Processed data checksum failed to decompressed to the expected chunk uncompressed length. (was {writtenDecompressed}, should be {info.UncompressedLength})" );
}
- var dataCrc = Utils.AdlerHash( processedData );
+ var dataCrc = Utils.AdlerHash( destination.AsSpan()[ ..writtenDecompressed ] );
- if ( dataCrc != ChunkInfo.Checksum )
+ if ( dataCrc != info.Checksum )
{
throw new InvalidDataException( "Processed data checksum is incorrect! Downloaded depot chunk is corrupt or invalid/wrong depot key?" );
}
- Data = processedData;
- IsProcessed = true;
+ return writtenDecompressed;
}
}
}
diff --git a/SteamKit2/SteamKit2/Types/DepotManifest.cs b/SteamKit2/SteamKit2/Types/DepotManifest.cs
index 891abd371..ff877eebb 100644
--- a/SteamKit2/SteamKit2/Types/DepotManifest.cs
+++ b/SteamKit2/SteamKit2/Types/DepotManifest.cs
@@ -168,10 +168,18 @@ internal FileData(string filename, byte[] filenameHash, EDepotFileFlag flag, ulo
///
public uint EncryptedCRC { get; private set; }
-
- internal DepotManifest(byte[] data)
+ ///
+ /// Initializes a new instance of the class.
+ /// Depot manifests may come from the Steam CDN or from Steam/depotcache/ manifest files.
+ ///
+ /// Raw depot manifest stream to deserialize.
+ /// Thrown if the given data is not something recognizable.
+ /// Thrown if the given data is not complete.
+ public static DepotManifest Deserialize( Stream stream )
{
- InternalDeserialize(data);
+ var manifest = new DepotManifest();
+ manifest.InternalDeserialize( stream );
+ return manifest;
}
///
@@ -180,7 +188,12 @@ internal DepotManifest(byte[] data)
///
/// Raw depot manifest data to deserialize.
/// Thrown if the given data is not something recognizable.
- public static DepotManifest Deserialize(byte[] data) => new(data);
+ /// Thrown if the given data is not complete.
+ public static DepotManifest Deserialize( byte[] data )
+ {
+ using var ms = new MemoryStream( data );
+ return Deserialize( ms );
+ }
///
/// Attempts to decrypts file names with the given encryption key.
@@ -284,63 +297,63 @@ public void SaveToFile( string filename )
///
/// Input file name.
/// DepotManifest object if deserialization was successful; otherwise, null.
+ /// Thrown if the given data is not something recognizable.
+ /// Thrown if the given data is not complete.
public static DepotManifest? LoadFromFile( string filename )
{
if ( !File.Exists( filename ) )
return null;
using var fs = File.Open( filename, FileMode.Open );
- using var ms = new MemoryStream();
- fs.CopyTo( ms );
- return Deserialize( ms.ToArray() );
+ return Deserialize( fs );
}
- void InternalDeserialize(byte[] data)
+ void InternalDeserialize( Stream stream )
{
ContentManifestPayload? payload = null;
ContentManifestMetadata? metadata = null;
ContentManifestSignature? signature = null;
- using ( var ms = new MemoryStream( data ) )
- using ( var br = new BinaryReader( ms ) )
+ using var br = new BinaryReader( stream, Encoding.UTF8, leaveOpen: true );
+
+ while ( true )
{
- while ( ( ms.Length - ms.Position ) > 0 )
+ uint magic = br.ReadUInt32();
+
+ if ( magic == DepotManifest.PROTOBUF_ENDOFMANIFEST_MAGIC )
{
- uint magic = br.ReadUInt32();
+ break;
+ }
- switch ( magic )
- {
- case Steam3Manifest.MAGIC:
- ms.Seek(-4, SeekOrigin.Current);
- Steam3Manifest binaryManifest = new Steam3Manifest( br );
- ParseBinaryManifest( binaryManifest );
-
- uint marker = br.ReadUInt32();
- if ( marker != magic )
- throw new InvalidDataException( "Unable to find end of message marker for depot manifest" );
- break;
-
- case DepotManifest.PROTOBUF_PAYLOAD_MAGIC:
- uint payload_length = br.ReadUInt32();
- payload = Serializer.Deserialize( ms, length: payload_length );
- break;
-
- case DepotManifest.PROTOBUF_METADATA_MAGIC:
- uint metadata_length = br.ReadUInt32();
- metadata = Serializer.Deserialize( ms, length: metadata_length );
- break;
-
- case DepotManifest.PROTOBUF_SIGNATURE_MAGIC:
- uint signature_length = br.ReadUInt32();
- signature = Serializer.Deserialize( ms, length: signature_length );
- break;
-
- case DepotManifest.PROTOBUF_ENDOFMANIFEST_MAGIC:
- break;
-
- default:
- throw new InvalidDataException( $"Unrecognized magic value {magic:X} in depot manifest." );
- }
+ switch ( magic )
+ {
+ case Steam3Manifest.MAGIC:
+ Steam3Manifest binaryManifest = new Steam3Manifest();
+ binaryManifest.Deserialize( br );
+ ParseBinaryManifest( binaryManifest );
+
+ uint marker = br.ReadUInt32();
+ if ( marker != magic )
+ throw new InvalidDataException( "Unable to find end of message marker for depot manifest" );
+ break;
+
+ case DepotManifest.PROTOBUF_PAYLOAD_MAGIC:
+ uint payload_length = br.ReadUInt32();
+ payload = Serializer.Deserialize( stream, length: payload_length );
+ break;
+
+ case DepotManifest.PROTOBUF_METADATA_MAGIC:
+ uint metadata_length = br.ReadUInt32();
+ metadata = Serializer.Deserialize( stream, length: metadata_length );
+ break;
+
+ case DepotManifest.PROTOBUF_SIGNATURE_MAGIC:
+ uint signature_length = br.ReadUInt32();
+ signature = Serializer.Deserialize( stream, length: signature_length );
+ break;
+
+ default:
+ throw new InvalidDataException( $"Unrecognized magic value {magic:X} in depot manifest." );
}
}
diff --git a/SteamKit2/SteamKit2/Types/Manifest.cs b/SteamKit2/SteamKit2/Types/Manifest.cs
index ed11e8708..761623ee9 100644
--- a/SteamKit2/SteamKit2/Types/Manifest.cs
+++ b/SteamKit2/SteamKit2/Types/Manifest.cs
@@ -110,33 +110,17 @@ internal void Deserialize( BinaryReader ds )
[NotNull]
public List? Mapping { get; private set; }
- public Steam3Manifest(byte[] data)
- {
- ArgumentNullException.ThrowIfNull( data );
-
- Deserialize(data);
- }
-
- internal Steam3Manifest(BinaryReader data)
- {
- Deserialize(data);
- }
-
- void Deserialize(byte[] data)
- {
- using var ms = new MemoryStream( data );
- using var br = new BinaryReader( ms );
- Deserialize( br );
- }
-
- void Deserialize( BinaryReader ds )
+ internal void Deserialize( BinaryReader ds )
{
+ /*
+ // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking
Magic = ds.ReadUInt32();
if (Magic != MAGIC)
{
throw new InvalidDataException("data is not a valid steam3 manifest: incorrect magic.");
}
+ */
Version = ds.ReadUInt32();
diff --git a/SteamKit2/SteamKit2/Util/CryptoHelper.cs b/SteamKit2/SteamKit2/Util/CryptoHelper.cs
index cbebe5e76..f14a63394 100644
--- a/SteamKit2/SteamKit2/Util/CryptoHelper.cs
+++ b/SteamKit2/SteamKit2/Util/CryptoHelper.cs
@@ -18,14 +18,12 @@ public static class CryptoHelper
///
/// Decrypts using AES/CBC/PKCS7 with an input byte array and key, using the random IV prepended using AES/ECB/None
///
- public static byte[] SymmetricDecrypt( byte[] input, byte[] key )
+ public static byte[] SymmetricDecrypt( ReadOnlySpan input, byte[] key )
{
- ArgumentNullException.ThrowIfNull( input );
ArgumentNullException.ThrowIfNull( key );
DebugLog.Assert( key.Length == 32, nameof( CryptoHelper ), $"{nameof( SymmetricDecrypt )} used with non 32 byte key!" );
- var inputSpan = input.AsSpan();
using var aes = Aes.Create();
aes.BlockSize = 128;
aes.KeySize = 256;
@@ -33,9 +31,9 @@ public static byte[] SymmetricDecrypt( byte[] input, byte[] key )
// first 16 bytes of input is the ECB encrypted IV
Span iv = stackalloc byte[ 16 ];
- aes.DecryptEcb( inputSpan[ ..iv.Length ], iv, PaddingMode.None );
+ aes.DecryptEcb( input[ ..iv.Length ], iv, PaddingMode.None );
- return aes.DecryptCbc( inputSpan[ iv.Length.. ], iv, PaddingMode.PKCS7 );
+ return aes.DecryptCbc( input[ iv.Length.. ], iv, PaddingMode.PKCS7 );
}
}
}
diff --git a/SteamKit2/SteamKit2/Util/LZMA/Compress/LZ/LzOutWindow.cs b/SteamKit2/SteamKit2/Util/LZMA/Compress/LZ/LzOutWindow.cs
index d479dbdcc..4616c4724 100644
--- a/SteamKit2/SteamKit2/Util/LZMA/Compress/LZ/LzOutWindow.cs
+++ b/SteamKit2/SteamKit2/Util/LZMA/Compress/LZ/LzOutWindow.cs
@@ -14,6 +14,14 @@ class OutWindow
public uint TrainSize = 0;
+ internal void SteamKitSetBuffer(byte[] buffer, uint windowSize) // Added by SteamKit to avoid allocating a byte array
+ {
+ _buffer = buffer;
+ _windowSize = windowSize;
+ _pos = 0;
+ _streamPos = 0;
+ }
+
public void Create(uint windowSize)
{
if (_windowSize != windowSize)
diff --git a/SteamKit2/SteamKit2/Util/LZMA/Compress/LZMA/LzmaDecoder.cs b/SteamKit2/SteamKit2/Util/LZMA/Compress/LZMA/LzmaDecoder.cs
index d9f835b82..7d2b9bd87 100644
--- a/SteamKit2/SteamKit2/Util/LZMA/Compress/LZMA/LzmaDecoder.cs
+++ b/SteamKit2/SteamKit2/Util/LZMA/Compress/LZMA/LzmaDecoder.cs
@@ -374,6 +374,20 @@ public void SetDecoderProperties(byte[] properties)
SetPosBitsProperties(pb);
}
+ public void SteamKitSetDecoderProperties(byte bits, uint dictionarySize, byte[] buffer) // Added by SteamKit to avoid allocating the OutWindow
+ {
+ int lc = bits % 9;
+ int remainder = bits / 9;
+ int lp = remainder % 5;
+ int pb = remainder / 5;
+ if (pb > Base.kNumPosStatesBitsMax || dictionarySize < (1 << 12))
+ throw new InvalidParamException();
+ SetLiteralProperties(lp, lc);
+ SetPosBitsProperties(pb);
+ m_DictionarySize = m_DictionarySizeCheck = dictionarySize;
+ m_OutWindow.SteamKitSetBuffer(buffer, dictionarySize);
+ }
+
public bool Train(System.IO.Stream stream)
{
_solid = true;
diff --git a/SteamKit2/SteamKit2/Util/Utils.cs b/SteamKit2/SteamKit2/Util/Utils.cs
index 9ed50bd41..dc450171a 100644
--- a/SteamKit2/SteamKit2/Util/Utils.cs
+++ b/SteamKit2/SteamKit2/Util/Utils.cs
@@ -17,10 +17,8 @@ static class Utils
///
/// Performs an Adler32 on the given input
///
- public static uint AdlerHash( byte[] input )
+ public static uint AdlerHash( ReadOnlySpan input )
{
- ArgumentNullException.ThrowIfNull( input );
-
uint a = 0, b = 0;
for ( int i = 0; i < input.Length; i++ )
{
diff --git a/SteamKit2/SteamKit2/Util/VZipUtil.cs b/SteamKit2/SteamKit2/Util/VZipUtil.cs
index ddfad5da4..dd2e2dea2 100644
--- a/SteamKit2/SteamKit2/Util/VZipUtil.cs
+++ b/SteamKit2/SteamKit2/Util/VZipUtil.cs
@@ -1,4 +1,5 @@
using System;
+using System.Buffers;
using System.IO;
using System.IO.Hashing;
@@ -13,9 +14,8 @@ static class VZipUtil
private const char Version = 'a';
- public static byte[] Decompress(byte[] buffer)
+ public static int Decompress( MemoryStream ms, byte[] destination, bool verifyChecksum = true )
{
- using MemoryStream ms = new MemoryStream( buffer );
using BinaryReader reader = new BinaryReader( ms );
if ( reader.ReadUInt16() != VZipHeader )
{
@@ -29,9 +29,12 @@ public static byte[] Decompress(byte[] buffer)
// Sometimes this is a creation timestamp (e.g. for Steam Client VZips).
// Sometimes this is a CRC32 (e.g. for depot chunks).
- /* uint creationTimestampOrSecondaryCRC = */ reader.ReadUInt32();
+ /* uint creationTimestampOrSecondaryCRC = */
+ reader.ReadUInt32();
- var properties = reader.ReadBytes( 5 );
+ // this is 5 bytes of LZMA properties
+ var propertyBits = reader.ReadByte();
+ var dictionarySize = reader.ReadUInt32();
var compressedBytesOffset = ms.Position;
// jump to the end of the buffer to read the footer
@@ -45,26 +48,41 @@ public static byte[] Decompress(byte[] buffer)
throw new Exception( "Expecting VZipFooter at end of stream" );
}
+ if ( destination.Length < sizeDecompressed )
+ {
+ throw new ArgumentException( "The destination buffer is smaller than the decompressed data size.", nameof( destination ) );
+ }
+
// jump back to the beginning of the compressed data
ms.Position = compressedBytesOffset;
SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder();
- decoder.SetDecoderProperties( properties );
+ // If the value of dictionary size in properties is smaller than (1 << 12),
+ // the LZMA decoder must set the dictionary size variable to (1 << 12).
+ var windowBuffer = ArrayPool.Shared.Rent( Math.Max( 1 << 12, ( int )dictionarySize ) );
+
+ try
+ {
+ decoder.SteamKitSetDecoderProperties( propertyBits, dictionarySize, windowBuffer );
- var outData = new byte[ sizeDecompressed ];
- using MemoryStream outStream = new MemoryStream( outData );
- decoder.Code( ms, outStream, sizeCompressed, sizeDecompressed, null );
+ using MemoryStream outStream = new MemoryStream( destination );
+ decoder.Code( ms, outStream, sizeCompressed, sizeDecompressed, null );
+ }
+ finally
+ {
+ ArrayPool.Shared.Return( windowBuffer );
+ }
- if ( Crc32.HashToUInt32( outData ) != outputCRC )
+ if ( verifyChecksum && Crc32.HashToUInt32( destination.AsSpan()[ ..sizeDecompressed ] ) != outputCRC )
{
throw new InvalidDataException( "CRC does not match decompressed data. VZip data may be corrupted." );
}
- return outData;
+ return sizeDecompressed;
}
- public static byte[] Compress(byte[] buffer)
+ public static byte[] Compress( byte[] buffer )
{
using MemoryStream ms = new MemoryStream();
using BinaryWriter writer = new BinaryWriter( ms );
diff --git a/SteamKit2/SteamKit2/Util/ZipUtil.cs b/SteamKit2/SteamKit2/Util/ZipUtil.cs
index 0ccdec8c9..f579f5e13 100644
--- a/SteamKit2/SteamKit2/Util/ZipUtil.cs
+++ b/SteamKit2/SteamKit2/Util/ZipUtil.cs
@@ -14,29 +14,31 @@ namespace SteamKit2
{
static class ZipUtil
{
- public static byte[] Decompress( byte[] buffer )
+ public static int Decompress( MemoryStream ms, byte[] destination, bool verifyChecksum = true )
{
- using var ms = new MemoryStream( buffer );
using var zip = new ZipArchive( ms );
var entries = zip.Entries;
DebugLog.Assert( entries.Count == 1, nameof( ZipUtil ), "Expected the zip to contain only one file" );
var entry = entries[ 0 ];
- var decompressed = new byte[ entry.Length ];
+ var sizeDecompressed = ( int )entry.Length;
+
+ if ( destination.Length < sizeDecompressed )
+ {
+ throw new ArgumentException( "The destination buffer is smaller than the decompressed data size.", nameof( destination ) );
+ }
using var entryStream = entry.Open();
- using var entryMemory = new MemoryStream( decompressed );
- entryStream.CopyTo( entryMemory );
- var checkSum = Crc32.HashToUInt32( decompressed );
+ entryStream.ReadExactly( destination, 0, sizeDecompressed );
- if ( checkSum != entry.Crc32 )
+ if ( verifyChecksum && Crc32.HashToUInt32( destination.AsSpan()[ ..sizeDecompressed ] ) != entry.Crc32 )
{
throw new Exception( "Checksum validation failed for decompressed file" );
}
- return decompressed;
+ return sizeDecompressed;
}
}
}
diff --git a/SteamKit2/Tests/CDNClientFacts.cs b/SteamKit2/Tests/CDNClientFacts.cs
index bc764cc35..45ec58aa2 100644
--- a/SteamKit2/Tests/CDNClientFacts.cs
+++ b/SteamKit2/Tests/CDNClientFacts.cs
@@ -44,10 +44,10 @@ public async Task ThrowsSteamKitWebExceptionOnUnsuccessfulWebResponseForChunk()
};
var chunk = new DepotManifest.ChunkData
{
- ChunkID = [0xFF],
+ ChunkID = [ 0xFF ],
};
- var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server ) );
+ var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server, [] ) );
Assert.Equal( ( HttpStatusCode )418, ex.StatusCode );
}
@@ -66,10 +66,58 @@ public async Task ThrowsWhenNoChunkIDIsSet()
};
var chunk = new DepotManifest.ChunkData();
- var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server ) );
+ var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server, [] ) );
Assert.Equal( "chunk", ex.ParamName );
}
+ [Fact]
+ public async Task ThrowsWhenDestinationBufferSmaller()
+ {
+ var configuration = SteamConfiguration.Create( x => x.WithHttpClientFactory( () => new HttpClient( new TeapotHttpMessageHandler() ) ) );
+ var steam = new SteamClient( configuration );
+ using var client = new Client( steam );
+ var server = new Server
+ {
+ Protocol = Server.ConnectionProtocol.HTTP,
+ Host = "localhost",
+ VHost = "localhost",
+ Port = 80
+ };
+ var chunk = new DepotManifest.ChunkData
+ {
+ ChunkID = [ 0xFF ],
+ UncompressedLength = 64,
+ CompressedLength = 32,
+ };
+
+ var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server, new byte[ 4 ] ) );
+ Assert.Equal( "destination", ex.ParamName );
+ }
+
+ [Fact]
+ public async Task ThrowsWhenDestinationBufferSmallerWithDepotKey()
+ {
+ var configuration = SteamConfiguration.Create( x => x.WithHttpClientFactory( () => new HttpClient( new TeapotHttpMessageHandler() ) ) );
+ var steam = new SteamClient( configuration );
+ using var client = new Client( steam );
+ var server = new Server
+ {
+ Protocol = Server.ConnectionProtocol.HTTP,
+ Host = "localhost",
+ VHost = "localhost",
+ Port = 80
+ };
+ var chunk = new DepotManifest.ChunkData
+ {
+ ChunkID = [ 0xFF ],
+ UncompressedLength = 64,
+ CompressedLength = 32,
+ };
+
+ var ex = await Assert.ThrowsAsync( () => client.DownloadDepotChunkAsync( depotId: 0, chunk, server, new byte[ 4 ], depotKey: [] ) );
+ Assert.Equal( "destination", ex.ParamName );
+ }
+
sealed class TeapotHttpMessageHandler : HttpMessageHandler
{
protected override Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken )
diff --git a/SteamKit2/Tests/DepotManifestFacts.cs b/SteamKit2/Tests/DepotManifestFacts.cs
index 7a10a9559..ea7e90c19 100644
--- a/SteamKit2/Tests/DepotManifestFacts.cs
+++ b/SteamKit2/Tests/DepotManifestFacts.cs
@@ -14,14 +14,12 @@ public class DepotManifestFacts
public void ParsesAndDecryptsManifest()
{
var assembly = Assembly.GetExecutingAssembly();
- using var stream = assembly.GetManifestResourceStream( "Tests.Files.depot_440_1118032470228587934.zip" );
+ using var stream = assembly.GetManifestResourceStream( "Tests.Files.depot_440_1118032470228587934.manifest" );
using var ms = new MemoryStream();
stream.CopyTo( ms );
var manifestData = ms.ToArray();
- manifestData = ZipUtil.Decompress( manifestData );
-
var depotManifest = DepotManifest.Deserialize( manifestData );
Assert.True( depotManifest.FilenamesEncrypted );
@@ -55,14 +53,12 @@ public void ParsesDecryptedManifest()
public void RoundtripSerializesManifestEncryptedManifest()
{
var assembly = Assembly.GetExecutingAssembly();
- using var stream = assembly.GetManifestResourceStream( "Tests.Files.depot_440_1118032470228587934.zip" );
+ using var stream = assembly.GetManifestResourceStream( "Tests.Files.depot_440_1118032470228587934.manifest" );
using var ms = new MemoryStream();
stream.CopyTo( ms );
var manifestData = ms.ToArray();
- manifestData = ZipUtil.Decompress( manifestData );
-
var depotManifest = DepotManifest.Deserialize( manifestData );
using var actualStream = new MemoryStream();
diff --git a/SteamKit2/Tests/Files/depot_440_1118032470228587934.manifest b/SteamKit2/Tests/Files/depot_440_1118032470228587934.manifest
new file mode 100644
index 000000000..da6770ec1
Binary files /dev/null and b/SteamKit2/Tests/Files/depot_440_1118032470228587934.manifest differ
diff --git a/SteamKit2/Tests/Files/depot_440_1118032470228587934.zip b/SteamKit2/Tests/Files/depot_440_1118032470228587934.zip
deleted file mode 100644
index f329c3c8c..000000000
Binary files a/SteamKit2/Tests/Files/depot_440_1118032470228587934.zip and /dev/null differ