Merge pull request #369 from /issues/364-BenchmarkSetupMemoryCheck

SteamPrefill.sln.DotSettings

@@ -27,6 +27,7 @@
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpUseContinuousIndentInsideBracesMigration/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateBlankLinesAroundFieldToBlankLinesAroundProperty/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EPredefinedNamingRulesToUserRulesUpgrade/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Bitrate/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=cacheable/@EntryIndexedValue">True</s:Boolean>
 
 	<!-- Custom spell check dictionary entries -->

SteamPrefill/CliCommands/Benchmark/BenchmarkModels.cs

@@ -15,18 +15,9 @@ public sealed class BenchmarkWorkload
 
         public List<QueuedRequest> AllQueuedRequests => QueuedAppsList.SelectMany(e => e.QueuedRequests).ToList();
 
-        private ByteSize? _totalDownloadSize;
-        public ByteSize TotalDownloadSize
-        {
-            get
-            {
-                if (_totalDownloadSize == null)
-                {
-                    _totalDownloadSize = ByteSize.FromBytes(QueuedAppsList.Sum(e => e.TotalBytes));
-                }
-                return _totalDownloadSize.Value;
-            }
-        }
+        private ByteSize _totalDownloadSize;
+        public ByteSize TotalDownloadSize => _totalDownloadSize;
+        public string TotalDownloadSizeFormatted => _totalDownloadSize.ToDecimalString();
 
         public long TotalFiles => QueuedAppsList.Sum(e => e.FileCount);
         public string TotalFilesFormatted => TotalFiles.ToString("n0");
@@ -43,6 +34,7 @@ public ByteSize TotalDownloadSize
         public BenchmarkWorkload(ConcurrentBag<AppQueuedRequests> queuedAppsList, ConcurrentStack<Server> cdnServers)
         {
             QueuedAppsList = queuedAppsList;
+            _totalDownloadSize = ByteSize.FromBytes(QueuedAppsList.Sum(e => e.TotalBytes));
             ServerShimList = cdnServers.Select(e => _mapper.Map<Server, CdnServerShim>(e)).ToList();
         }
 

SteamPrefill/Extensions/SystemMemoryMetrics.cs

@@ -0,0 +1,23 @@
+namespace SteamPrefill.Extensions
+{
+    public static class SystemMemoryMetrics
+    {
+        public static ByteSize GetTotalSystemMemory()
+        {
+            var info = new ProcessStartInfo("free -m")
+            {
+                FileName = "/bin/bash",
+                Arguments = "-c \"free -m\"",
+                RedirectStandardOutput = true
+            };
+
+            using var process = Process.Start(info);
+            var output = process.StandardOutput.ReadToEnd();
+
+            var lines = output.Split("\n");
+            var memory = lines[1].Split(" ", StringSplitOptions.RemoveEmptyEntries);
+
+            return ByteSize.FromBytes(double.Parse(memory[1]) * ByteSize.FromMebiBytes(1).Bytes);
+        }
+    }
+}

SteamPrefill/Properties/GlobalUsings.cs

@@ -36,6 +36,7 @@
 global using System.Linq;
 global using System.Net;
 global using System.Net.Http;
+global using System.Runtime.InteropServices;
 global using System.Threading;
 global using System.Text.Json;
 global using System.Text.Json.Serialization;

SteamPrefill/SteamManager.cs

@@ -255,10 +255,55 @@ public async Task SetupBenchmarkAsync(List<uint> appIds, bool useAllOwnedGames,
             // Writing stats
             benchmarkWorkload.PrintSummary(_ansiConsole);
 
-            var fileSize = ByteSize.FromBytes(new FileInfo(AppConfig.BenchmarkWorkloadPath).Length);
+            CheckBenchmarkWorkloadSize(benchmarkWorkload);
+
             _ansiConsole.Write(new Rule());
             _ansiConsole.LogMarkupLine("Completed build of workload file...");
-            _ansiConsole.LogMarkupLine($"Resulting file size : {MediumPurple(fileSize.ToBinaryString())}");
+        }
+
+        //TODO document
+        private static void CheckBenchmarkWorkloadSize(BenchmarkWorkload generatedWorkload)
+        {
+            // While technically you can run a benchmark from a client Linux machine, this scenario is probably pretty rare since most people
+            // run SteamPrefill on the cache host.  Rather than add some more complexity by checking that the benchmark is definitely being created
+            // on the cache host, we'll omit it and save some added complexity
+            if (!System.OperatingSystem.IsLinux())
+            {
+                return;
+            }
+
+            var systemMemory = SystemMemoryMetrics.GetTotalSystemMemory();
+            // If the user generated a workload that is larger than the system's total memory, then the benchmark will always be reading from disk.
+            if (generatedWorkload.TotalDownloadSize > systemMemory)
+            {
+                return;
+            }
+
+            // Table setup
+            var table = new Table
+            {
+                ShowHeaders = false,
+                Border = TableBorder.Rounded,
+                BorderStyle = new Style(SpectreColors.LightYellow)
+            };
+            table.AddColumn("");
+
+            // Adding message rows
+            table.AddRow(LightYellow($"{new string(' ', 40)}!!!!!! Warning !!!!!!"));
+            table.AddEmptyRow();
+            table.AddRow($"The generated workload size of {Magenta(generatedWorkload.TotalDownloadSizeFormatted)} " +
+                         $"is smaller than the total system memory of {LightYellow(systemMemory.ToDecimalString())}.");
+            table.AddRow("Linux will cache files that it reads in system memory to improve performance of frequently used files,");
+            table.AddRow("however this benchmark is typically used to test disk IO performance.");
+            table.AddRow("In order to guarantee that an accurate benchmark where files are only ever read from disk,");
+            table.AddRow("the workload size should be larger than the system's memory.");
+            table.AddEmptyRow();
+            table.AddRow($"Please create a new benchmark workload that is larger than {LightYellow(systemMemory.ToDecimalString())}.");
+
+
+            // Render the table to the console
+            AnsiConsole.Write(table);
+            AnsiConsole.WriteLine();
         }
 
         //TODO this method is awfully complex, has lots and lots of nesting.  Makes it a bit hard to read at a glance
@@ -378,6 +423,7 @@ await _ansiConsole.CreateSpectreProgress().StartAsync(async ctx =>
             _ansiConsole.Write(new Rule());
         }
 
+        //TODO rename
         private IOrderedEnumerable<KeyValuePair<string, ByteSize>> SortData(ConcurrentDictionary<string, ByteSize> index, SortOrder sortOrder, SortColumn sortColumn)
         {
             if (sortOrder == SortOrder.Ascending)
@@ -399,7 +445,6 @@ private IOrderedEnumerable<KeyValuePair<string, ByteSize>> SortData(ConcurrentDi
 
         #endregion
 
-
         public async Task<List<AppInfo>> GetAllAvailableAppsAsync()
         {
             var ownedGameIds = _steam3.LicenseManager.AllOwnedAppIds;