Skip to content

Commit

Permalink
Use Steam manifest format
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicknineTheEagle
NicknineTheEagle committed Oct 21, 2024
1 parent 9ed8a70 commit a394bc6
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 99 deletions.
133 changes: 37 additions & 96 deletions DepotDownloader/ContentDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,20 +591,20 @@ static async Task<DepotDownloadInfo> GetDepotInfo(uint depotId, uint appId, ulon
return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
}

private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
{
public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
public DepotManifest.ChunkData OldChunk { get; } = oldChunk;
public DepotManifest.ChunkData NewChunk { get; } = newChunk;
}

private class DepotFilesData
{
public DepotDownloadInfo depotDownloadInfo;
public DepotDownloadCounter depotCounter;
public string stagingDir;
public ProtoManifest manifest;
public ProtoManifest previousManifest;
public List<ProtoManifest.FileData> filteredFiles;
public DepotManifest manifest;
public DepotManifest previousManifest;
public List<DepotManifest.FileData> filteredFiles;
public HashSet<string> allFileNames;
}

Expand Down Expand Up @@ -687,8 +687,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

Console.WriteLine("Processing depot {0}", depot.DepotId);

ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
DepotManifest oldManifest = null;
DepotManifest newManifest = null;
var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);

var lastManifestId = INVALID_MANIFEST_ID;
Expand All @@ -700,72 +700,28 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));

if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum;

try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}

oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);

if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
oldProtoManifest = null;
}
}
// We only have to show this warning if the old manifest ID was different
var badHashWarning = (lastManifestId != depot.ManifestId);
oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning);
}

if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
if (lastManifestId == depot.ManifestId && oldManifest != null)
{
newProtoManifest = oldProtoManifest;
newManifest = oldManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum;
newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true);

try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}

newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);

if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}

if (newProtoManifest != null)
if (newManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
Console.Write("Downloading depot manifest... ");

DepotManifest depotManifest = null;
ulong manifestRequestCode = 0;
var manifestRequestCodeExpiration = DateTime.MinValue;

Expand Down Expand Up @@ -813,7 +769,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depot.ManifestId,
connection,
cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
newManifest = await cdnPool.CDNClient.DownloadManifestAsync(
depot.DepotId,
depot.ManifestId,
manifestRequestCode,
Expand Down Expand Up @@ -865,9 +821,9 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
}
} while (depotManifest == null);
} while (newManifest == null);

if (depotManifest == null)
if (newManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
cts.Cancel();
Expand All @@ -876,28 +832,22 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();


newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);

Util.SaveManifestToFile(configDir, newManifest);
Console.WriteLine(" Done!");
}
}

newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));

Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime);

if (Config.DownloadManifestOnly)
{
DumpManifestToTextFile(depot, newProtoManifest);
DumpManifestToTextFile(depot, newManifest);
return null;
}

var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);

var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);

// Pre-process
Expand Down Expand Up @@ -929,8 +879,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
manifest = newManifest,
previousManifest = oldManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
};
Expand All @@ -945,7 +895,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource
Console.WriteLine("Downloading depot {0}", depot.DepotId);

var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>();

await Util.InvokeAsync(
files.Select(file => new Func<Task>(async () =>
Expand Down Expand Up @@ -999,16 +949,16 @@ private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
DepotManifest.FileData file,
ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue)
{
cts.Token.ThrowIfCancellationRequested();

var depot = depotFilesData.depotDownloadInfo;
var stagingDir = depotFilesData.stagingDir;
var depotDownloadCounter = depotFilesData.depotCounter;
var oldProtoManifest = depotFilesData.previousManifest;
ProtoManifest.FileData oldManifestFile = null;
DepotManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
Expand All @@ -1023,7 +973,7 @@ private static void DownloadSteam3AsyncDepotFile(
File.Delete(fileStagingPath);
}

List<ProtoManifest.ChunkData> neededChunks;
List<DepotManifest.ChunkData> neededChunks;
var fi = new FileInfo(fileFinalPath);
var fileDidExist = fi.Exists;
if (!fileDidExist)
Expand All @@ -1041,7 +991,7 @@ private static void DownloadSteam3AsyncDepotFile(
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
}

neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
}
else
{
Expand Down Expand Up @@ -1085,7 +1035,7 @@ private static void DownloadSteam3AsyncDepotFile(
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);

var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum)))
{
neededChunks.Add(match.NewChunk);
}
Expand Down Expand Up @@ -1204,9 +1154,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
DepotManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
DepotManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();

Expand All @@ -1215,17 +1165,8 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(

var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();

var data = new DepotManifest.ChunkData
{
ChunkID = chunk.ChunkID,
Checksum = BitConverter.ToUInt32(chunk.Checksum),
Offset = chunk.Offset,
CompressedLength = chunk.CompressedLength,
UncompressedLength = chunk.UncompressedLength
};

var written = 0;
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)chunk.UncompressedLength);

try
{
Expand All @@ -1249,7 +1190,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
depot.DepotId,
data,
chunk,
connection,
chunkBuffer,
depot.DepotKey,
Expand Down Expand Up @@ -1318,7 +1259,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
}

fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
}
finally
Expand Down Expand Up @@ -1362,7 +1303,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
}
}

static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest)
{
var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
using var sw = new StreamWriter(txtManifest);
Expand Down
35 changes: 35 additions & 0 deletions DepotDownloader/ProtoManifest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.IO;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
using ProtoBuf;
using SteamKit2;

Expand Down Expand Up @@ -157,5 +158,39 @@ public void SaveToFile(string filename, out byte[] checksum)
using var ds = new DeflateStream(fs, CompressionMode.Compress);
ms.CopyTo(ds);
}

public DepotManifest ConvertToSteamManifest(uint depotId)
{
ulong uncompressedSize = 0, compressedSize = 0;
var newManifest = new DepotManifest();
newManifest.Files = new List<DepotManifest.FileData>(Files.Count);

foreach (var file in Files)
{
var fileNameHash = SHA1.HashData(Encoding.UTF8.GetBytes(file.FileName.Replace('/', '\\').ToLowerInvariant()));
var newFile = new DepotManifest.FileData(file.FileName, fileNameHash, file.Flags, file.TotalSize, file.FileHash, null, false, file.Chunks.Count);

foreach (var chunk in file.Chunks)
{
var newChunk = new DepotManifest.ChunkData(chunk.ChunkID, BitConverter.ToUInt32(chunk.Checksum, 0), chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength);
newFile.Chunks.Add(newChunk);

uncompressedSize += chunk.UncompressedLength;
compressedSize += chunk.CompressedLength;
}

newManifest.Files.Add(newFile);
}

newManifest.FilenamesEncrypted = false;
newManifest.DepotID = depotId;
newManifest.ManifestGID = ID;
newManifest.CreationTime = CreationTime;
newManifest.TotalUncompressedSize = uncompressedSize;
newManifest.TotalCompressedSize = compressedSize;
newManifest.EncryptedCRC = 0;

return newManifest;
}
}
}
Loading

0 comments on commit a394bc6

Please sign in to comment.