Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support downloading manifests/chunks with missing Content-Length #1406

Merged
merged 2 commits into from
Aug 26, 2024
Merged

Support downloading manifests/chunks with missing Content-Length #1406

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fea7f2f
Fallback to chunk CompressedLength if Content-Length is not set
xPaw Aug 26, 2024
1cdf7a7
Support downloading manifests without Content-Length
xPaw Aug 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 51 additions & 23 deletions SteamKit2/SteamKit2/Steam/CDN/Client.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,30 +100,45 @@ public async Task<DepotManifest> DownloadManifestAsync( uint depotId, ulong mani
throw new SteamKitWebRequestException( $"Response status code does not indicate success: {response.StatusCode:D} ({response.ReasonPhrase}).", response );
}

if ( !response.Content.Headers.ContentLength.HasValue )
{
throw new SteamKitWebRequestException( "Response does not have Content-Length", response );
}

cts.CancelAfter( ResponseBodyTimeout );

var contentLength = ( int )response.Content.Headers.ContentLength;
var buffer = ArrayPool<byte>.Shared.Rent( contentLength );
var contentLength = -1;
byte[]? buffer = null;

if ( response.Content.Headers.ContentLength.HasValue )
{
contentLength = ( int )response.Content.Headers.ContentLength;
buffer = ArrayPool<byte>.Shared.Rent( contentLength );
}
else
{
DebugLog.WriteLine( nameof( CDN ), $"Manifest response does not have Content-Length, falling back to unbuffered read." );
}

try
{
using var ms = new MemoryStream( buffer, 0, contentLength );
MemoryStream ms;

// Stream the http response into the rented buffer
await response.Content.CopyToAsync( ms, cts.Token );
if ( buffer != null )
{
ms = new MemoryStream( buffer, 0, contentLength );

if ( ms.Position != contentLength )
// Stream the http response into the rented buffer
await response.Content.CopyToAsync( ms, cts.Token );

if ( ms.Position != contentLength )
{
throw new InvalidDataException( $"Length mismatch after downloading depot manifest! (was {ms.Position}, but should be {contentLength})" );
}

ms.Position = 0;
}
else
{
throw new InvalidDataException( $"Length mismatch after downloading depot manifest! (was {ms.Position}, but should be {contentLength})" );
var data = await response.Content.ReadAsByteArrayAsync();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we avoiding ReadAsStreamAsync because we need to be able to seek?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ZipArchive(Stream) is a lie because it will copy the stream into an internal MemoryStream if stream is not seekable: https://source.dot.net/#System.IO.Compression/System/IO/Compression/ZipArchive.cs,138

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol

ms = new MemoryStream( data );
}

ms.Position = 0;

// Decompress the zipped manifest data
using var zip = new ZipArchive( ms );
var entries = zip.Entries;
Expand All @@ -132,10 +147,15 @@ public async Task<DepotManifest> DownloadManifestAsync( uint depotId, ulong mani

using var zipEntryStream = entries[ 0 ].Open();
depotManifest = DepotManifest.Deserialize( zipEntryStream );

ms.Dispose();
}
finally
{
ArrayPool<byte>.Shared.Return( buffer );
if ( buffer != null )
{
ArrayPool<byte>.Shared.Return( buffer );
}
}
}
catch ( Exception ex )
Expand Down Expand Up @@ -224,17 +244,25 @@ public async Task<int> DownloadDepotChunkAsync( uint depotId, DepotManifest.Chun
throw new SteamKitWebRequestException( $"Response status code does not indicate success: {response.StatusCode:D} ({response.ReasonPhrase}).", response );
}

if ( !response.Content.Headers.ContentLength.HasValue )
{
throw new SteamKitWebRequestException( "Response does not have Content-Length", response );
}
var contentLength = ( int )chunk.CompressedLength;

var contentLength = ( int )response.Content.Headers.ContentLength;
if ( response.Content.Headers.ContentLength.HasValue )
{
contentLength = ( int )response.Content.Headers.ContentLength;

// assert that lengths match only if the chunk has a length assigned.
if ( chunk.CompressedLength > 0 && contentLength != chunk.CompressedLength )
// assert that lengths match only if the chunk has a length assigned.
if ( chunk.CompressedLength > 0 && contentLength != chunk.CompressedLength )
{
throw new InvalidDataException( $"Content-Length mismatch for depot chunk! (was {contentLength}, but should be {chunk.CompressedLength})" );
}
}
else if ( contentLength > 0 )
{
DebugLog.WriteLine( nameof( CDN ), $"Response does not have Content-Length, falling back to chunk.CompressedLength." );
}
else
{
throw new InvalidDataException( $"Content-Length mismatch for depot chunk! (was {contentLength}, but should be {chunk.CompressedLength})" );
throw new SteamKitWebRequestException( "Response does not have Content-Length and chunk.CompressedLength is not set.", response );
}

cts.CancelAfter( ResponseBodyTimeout );
Expand Down
Loading