Merge pull request #674 from alanmcgovern/readonly-hashing

Ensure hash checking a file is a purely read-only operation

src/MonoTorrent.Client/MonoTorrent.Client.Modes/HashingMode.cs

@@ -128,6 +128,8 @@ public async Task WaitForHashingToComplete ()
                 for (int i = 0; i < Manager.Torrent!.PieceCount; i++)
                     Manager.OnPieceHashed (i, false, i + 1, Manager.Torrent.PieceCount);
             }
+
+            await Manager.RefreshAllFilesCorrectLengthAsync ();
         }
 
         public void Dispose ()

src/MonoTorrent.Client/MonoTorrent.Client.Modes/StartingMode.cs

@@ -90,9 +90,11 @@ public async Task WaitForStartingToComplete ()
                 throw new TorrentException ("Torrents with no metadata must use 'MetadataMode', not 'StartingMode'.");
 
             try {
-                // If the torrent contains any files of length 0, ensure we create them now.
-                await CreateEmptyFiles ();
+                // Run some basic validations to see if we should force a hashcheck
                 await VerifyHashState ();
+
+                // Create any files of length zero, and truncate any files which are too long
+                await CreateOrTruncateFiles ();
                 Cancellation.Token.ThrowIfCancellationRequested ();
                 Manager.PieceManager.Initialise ();
             } catch (Exception ex) {
@@ -153,21 +155,33 @@ public async Task WaitForStartingToComplete ()
             await Manager.LocalPeerAnnounceAsync ();
         }
 
-        async ReusableTask CreateEmptyFiles ()
+        async ReusableTask CreateOrTruncateFiles ()
         {
-            foreach (var file in Manager.Files) {
-                if (file.Length == 0) {
-                    var fileInfo = new FileInfo (file.FullPath);
-                    if (fileInfo.Exists && fileInfo.Length == 0)
-                        continue;
-
-                    await MainLoop.SwitchToThreadpool ();
-                    Directory.CreateDirectory (Path.GetDirectoryName (file.FullPath)!);
-                    // Ensure file on disk is always 0 bytes, as it's supposed to be.
-                    using (var stream = File.OpenWrite (file.FullPath))
-                        stream.SetLength (0);
+            foreach (TorrentFileInfo file in Manager.Files.Where (t => t.Priority != Priority.DoNotDownload)) {
+                var maybeLength = await DiskManager.GetLengthAsync (file).ConfigureAwait (false);
+                // If the file doesn't exist, create it.
+                if (!maybeLength.HasValue)
+                    await DiskManager.CreateAsync (file, Settings.FileCreationOptions).ConfigureAwait (false);
+
+                // Otherwise if the destination file is larger than it should be, truncate it
+                else if (maybeLength.Value > file.Length)
+                    await DiskManager.SetLengthAsync (file, file.Length).ConfigureAwait (false);
+
+                if (file.Length == 0)
+                    file.BitField[0] = true;
+            }
+
+            // Then check if any 'DoNotDownload' file overlaps with a file we are downloading.
+            var downloadingFiles = Manager.Files.Where (t => t.Priority != Priority.DoNotDownload).ToArray ();
+            foreach (var ignoredFile in Manager.Files.Where (t => t.Priority == Priority.DoNotDownload)) {
+                foreach (var downloading in downloadingFiles) {
+                    if (ignoredFile.Overlaps (downloading))
+                        await DiskManager.CreateAsync (ignoredFile, Settings.FileCreationOptions);
                 }
             }
+
+            // After potentially creating or truncating files, refresh the state.
+            await Manager.RefreshAllFilesCorrectLengthAsync ();
         }
 
         async void SendAnnounces ()
@@ -206,6 +220,8 @@ async Task VerifyHashState ()
             if (!Manager.HashChecked)
                 return;
 
+            // Lightweight check - if any files are missing but we believe data should exist, reset the 'needs hashcheck' boolean
+            // so we force a hash check.
             foreach (ITorrentManagerFile file in Manager.Files) {
                 if (!file.BitField.AllFalse && file.Length > 0) {
                     if (!await DiskManager.CheckFileExistsAsync (file)) {

src/MonoTorrent.Client/MonoTorrent.Client/Managers/DiskManager.cs

@@ -720,5 +720,23 @@ internal async ReusableTask UpdateSettingsAsync (EngineSettings settings)
                 await Cache.SetCapacityAsync (settings.DiskCacheBytes);
             }
         }
+
+        internal async ReusableTask<bool> CreateAsync (ITorrentManagerFile file, FileCreationOptions fileCreationOptions)
+        {
+            await IOLoop;
+            return await Cache.Writer.CreateAsync (file, fileCreationOptions);
+        }
+
+        internal async ReusableTask<long?> GetLengthAsync (ITorrentManagerFile file)
+        {
+            await IOLoop;
+            return await Cache.Writer.GetLengthAsync (file);
+        }
+
+        internal async ReusableTask<bool> SetLengthAsync (ITorrentManagerFile file, long length)
+        {
+            await IOLoop;
+            return await Cache.Writer.SetLengthAsync (file, length);
+        }
     }
 }

src/MonoTorrent.Client/MonoTorrent.Client/Managers/TorrentManager.cs

@@ -87,17 +87,44 @@ public async Task SetFilePriorityAsync (ITorrentManagerFile file, Priority prior
             if (!Files.Contains (file))
                 throw new ArgumentNullException (nameof (file), "The file is not part of this torrent");
 
-            // No change
+            // No change - bail out
             if (priority == file.Priority)
                 return;
 
             await ClientEngine.MainLoop;
 
+            if (Engine == null)
+                throw new InvalidOperationException ("This torrent manager has been removed from it's ClientEngine");
+
             // If the old priority, or new priority, is 'DoNotDownload' then the selector needs to be refreshed
             bool needsToUpdateSelector = file.Priority == Priority.DoNotDownload || priority == Priority.DoNotDownload;
             var oldPriority = file.Priority;
+
+            if (oldPriority == Priority.DoNotDownload && !(await Engine.DiskManager.CheckFileExistsAsync (file))) {
+                // Always create the file the user requested to download
+                await Engine.DiskManager.CreateAsync (file, Engine.Settings.FileCreationOptions);
+
+                if (file.Length == 0)
+                    ((TorrentFileInfo) file).BitField[0] = await Engine.DiskManager.CheckFileExistsAsync (file);
+            }
+
+            // Update the priority for the file itself now that we've successfully created it!
             ((TorrentFileInfo) file).Priority = priority;
 
+            if (oldPriority == Priority.DoNotDownload && file.Length > 0) {
+                // Look for any file which are still marked DoNotDownload but also overlap this file.
+                // We need to create those ones too because if there are three 400kB files and the
+                // piece length is 512kB, and the first file is set to 'DoNotDownload', then we still
+                // need to create it as we'll download the first 512kB under bittorrent v1.
+                foreach (var maybeCreateFile in Files.Where (t => t.Priority == Priority.DoNotDownload && t.Length > 0)) {
+                    // If this file overlaps, create it!
+                    if (maybeCreateFile.Overlaps(file) && !(await Engine.DiskManager.CheckFileExistsAsync (maybeCreateFile)))
+                        await Engine.DiskManager.CreateAsync (maybeCreateFile, Engine.Settings.FileCreationOptions);
+                }
+            }
+;
+
+            // With the new priority, calculate which files we're actively downloading!
             if (needsToUpdateSelector) {
                 // If we change the priority of a file we need to figure out which files are marked
                 // as 'DoNotDownload' and which ones are downloadable.
@@ -154,7 +181,7 @@ public async Task SetFilePriorityAsync (ITorrentManagerFile file, Priority prior
         /// all files have the correct length. If some files are marked as 'DoNotDownload' then the
         /// torrent will not be considered to be Complete until they are downloaded.
         /// </summary>
-        public bool Complete => Bitfield.AllTrue;
+        public bool Complete => Bitfield.AllTrue && AllFilesCorrectLength;
 
         internal bool Disposed { get; private set; }
 
@@ -204,6 +231,7 @@ public async Task SetNeedsHashCheckAsync ()
         internal void SetNeedsHashCheck ()
         {
             HashChecked = false;
+            AllFilesCorrectLength = false;
             if (Engine != null && Engine.Settings.AutoSaveLoadFastResume) {
                 var path = Engine.Settings.GetFastResumePath (InfoHashes);
                 if (File.Exists (path))
@@ -220,6 +248,8 @@ internal void SetNeedsHashCheck ()
 
         public bool HashChecked { get; private set; }
 
+        internal bool AllFilesCorrectLength { get; private set; }
+
         /// <summary>
         /// The number of times a piece is downloaded, but is corrupt and fails the hashcheck and must be re-downloaded.
         /// </summary>
@@ -583,6 +613,7 @@ internal async Task HashCheckAsync (bool autoStart, bool setStoppedModeWhenDone)
             }
 
             HashChecked = true;
+
             if (autoStart) {
                 await StartAsync ();
             } else if (setStoppedModeWhenDone) {
@@ -680,8 +711,6 @@ internal void SetMetadata (Torrent torrent)
                 var currentPath = File.Exists (downloadCompleteFullPath) ? downloadCompleteFullPath : downloadIncompleteFullPath;
                 var torrentFileInfo = new TorrentFileInfo (file, currentPath);
                 torrentFileInfo.UpdatePaths ((currentPath, downloadCompleteFullPath, downloadIncompleteFullPath));
-                if (file.Length == 0)
-                    torrentFileInfo.BitField[0] = true;
                 return torrentFileInfo;
             }).Cast<ITorrentManagerFile> ().ToList ().AsReadOnly ();
 
@@ -1061,26 +1090,58 @@ void CheckRegisteredAndDisposed ()
                 throw new InvalidOperationException ("The registered engine has been disposed");
         }
 
-        public async Task LoadFastResumeAsync (FastResume data)
+        /// <summary>
+        /// Attempts to load the provided fastresume data. Several validations are performed during this, such as ensuring
+        /// files which have validated pieces actually exist on disk, and the length of those files is correct. If any validation
+        /// fails, the <see cref="HashChecked"/> boolean will not be set to true, and <see cref="HashCheckAsync(bool)"/> will need
+        /// to be run to re-averify the file contents.
+        /// </summary>
+        /// <param name="data"></param>
+        /// <returns></returns>
+        public Task LoadFastResumeAsync (FastResume data)
+            => LoadFastResumeAsync (data, false);
+
+        internal async Task LoadFastResumeAsync (FastResume data, bool skipStateCheckForTests)
         {
             if (data == null)
                 throw new ArgumentNullException (nameof (data));
 
             await ClientEngine.MainLoop;
 
             CheckMetadata ();
-            if (State != TorrentState.Stopped)
+            if (State != TorrentState.Stopped && !skipStateCheckForTests)
                 throw new InvalidOperationException ("Can only load FastResume when the torrent is stopped");
             if (InfoHashes != data.InfoHashes || Torrent!.PieceCount != data.Bitfield.Length)
                 throw new ArgumentException ("The fast resume data does not match this torrent", "fastResumeData");
 
             for (int i = 0; i < Torrent.PieceCount; i++)
                 OnPieceHashed (i, data.Bitfield[i], i + 1, Torrent.PieceCount);
+
             UnhashedPieces.From (data.UnhashedPieces);
 
+            await RefreshAllFilesCorrectLengthAsync ();
             HashChecked = true;
         }
 
+        internal async ReusableTask RefreshAllFilesCorrectLengthAsync ()
+        {
+            var allFilesCorrectLength = true;
+            foreach (TorrentFileInfo file in Files) {
+                var maybeLength = await Engine!.DiskManager.GetLengthAsync (file);
+
+                // Empty files aren't stored in fast resume data because it's as easy to just check if they exist on disk.
+                if (file.Length == 0)
+                    file.BitField[0] = maybeLength.HasValue;
+
+                // If any file doesn't exist, or any file is too large, indicate that something is wrong.
+                // If files exist but are too short, then we can assume everything is fine and the torrent just
+                // needs to be downloaded.
+                if (file.Priority != Priority.DoNotDownload && (!maybeLength.HasValue || maybeLength > file.Length))
+                    allFilesCorrectLength = false;
+            }
+            AllFilesCorrectLength = allFilesCorrectLength;
+        }
+
         public async Task<FastResume> SaveFastResumeAsync ()
         {
             await ClientEngine.MainLoop;

src/MonoTorrent.Client/MonoTorrent.Client/Settings/EngineSettings.cs

@@ -153,6 +153,11 @@ public sealed class EngineSettings : IEquatable<EngineSettings>
         /// </summary>
         public FastResumeMode FastResumeMode { get; } = FastResumeMode.BestEffort;
 
+        /// <summary>
+        /// Sets the preferred approach to creating new files.
+        /// </summary>
+        public FileCreationOptions FileCreationOptions { get; } = FileCreationOptions.PreferSparse;
+
         /// <summary>
         /// The list of HTTP(s) endpoints which the engine should bind to when a <see cref="TorrentManager"/> is set up
         /// to stream data from the torrent and <see cref="TorrentManager.StreamProvider"/> is non-null. Should be of
@@ -266,7 +271,8 @@ public EngineSettings ()
         internal EngineSettings (
             IList<EncryptionType> allowedEncryption, bool allowHaveSuppression, bool allowLocalPeerDiscovery, bool allowPortForwarding,
             bool autoSaveLoadDhtCache, bool autoSaveLoadFastResume, bool autoSaveLoadMagnetLinkMetadata, string cacheDirectory,
-            TimeSpan connectionTimeout, IPEndPoint? dhtEndPoint, int diskCacheBytes, CachePolicy diskCachePolicy, FastResumeMode fastResumeMode, Dictionary<string, IPEndPoint> listenEndPoints,
+            TimeSpan connectionTimeout, IPEndPoint? dhtEndPoint, int diskCacheBytes, CachePolicy diskCachePolicy, FastResumeMode fastResumeMode,
+            FileCreationOptions fileCreationMode, Dictionary<string, IPEndPoint> listenEndPoints,
             int maximumConnections, int maximumDiskReadRate, int maximumDiskWriteRate, int maximumDownloadRate, int maximumHalfOpenConnections,
             int maximumOpenFiles, int maximumUploadRate, IDictionary<string, IPEndPoint> reportedListenEndPoints, bool usePartialFiles,
             TimeSpan webSeedConnectionTimeout, TimeSpan webSeedDelay, int webSeedSpeedTrigger, TimeSpan staleRequestTimeout,
@@ -286,6 +292,7 @@ internal EngineSettings (
             CacheDirectory = cacheDirectory;
             ConnectionTimeout = connectionTimeout;
             FastResumeMode = fastResumeMode;
+            FileCreationOptions = fileCreationMode;
             HttpStreamingPrefix = httpStreamingPrefix;
             ListenEndPoints = new ReadOnlyDictionary<string, IPEndPoint> (new Dictionary<string, IPEndPoint> (listenEndPoints));
             MaximumConnections = maximumConnections;

src/MonoTorrent.Client/MonoTorrent.Client/Settings/EngineSettingsBuilder.cs

@@ -190,6 +190,11 @@ public int DiskCacheBytes {
         /// </summary>
         public FastResumeMode FastResumeMode { get; set; }
 
+        /// <summary>
+        /// Sets the preferred approach to creating new files.
+        /// </summary>
+        public FileCreationOptions FileCreationMode { get; set; }
+
         /// <summary>
         /// The HTTP(s) prefix which the engine should bind to when a <see cref="TorrentManager"/> is set up
         /// to stream data from the torrent and <see cref="TorrentManager.StreamProvider"/> is non-null. Should be of
@@ -355,6 +360,7 @@ public EngineSettingsBuilder (EngineSettings settings)
             DiskCacheBytes = settings.DiskCacheBytes;
             DiskCachePolicy = settings.DiskCachePolicy;
             FastResumeMode = settings.FastResumeMode;
+            FileCreationMode = settings.FileCreationOptions;
             httpStreamingPrefix = settings.HttpStreamingPrefix;
             ListenEndPoints = new Dictionary<string, IPEndPoint> (settings.ListenEndPoints);
             ReportedListenEndPoints = new Dictionary<string, IPEndPoint> (settings.ReportedListenEndPoints);
@@ -395,6 +401,7 @@ public EngineSettings ToSettings ()
                 diskCacheBytes: DiskCacheBytes,
                 diskCachePolicy: DiskCachePolicy,
                 fastResumeMode: FastResumeMode,
+                fileCreationMode: FileCreationMode,
                 httpStreamingPrefix: HttpStreamingPrefix,
                 listenEndPoints: ListenEndPoints,
                 maximumConnections: MaximumConnections,

src/MonoTorrent.Client/MonoTorrent.Client/Settings/Serializer.cs

@@ -77,6 +77,8 @@ static T Deserialize<T> (BEncodedDictionary dict)
                     property.SetValue (builder, new Uri (((BEncodedString) value).Text));
                 } else if (property.PropertyType == typeof (IPAddress)) {
                     property.SetValue (builder, IPAddress.Parse (((BEncodedString) value).Text));
+                } else if (property.PropertyType == typeof(FileCreationOptions)) {
+                    property.SetValue (builder, Enum.Parse (typeof (FileCreationOptions), ((BEncodedString) value).Text));
                 } else if (property.PropertyType == typeof (IPEndPoint)) {
                     var list = (BEncodedList) value;
                     IPEndPoint? endPoint = null;
@@ -115,6 +117,7 @@ static BEncodedDictionary Serialize (object builder)
                     FastResumeMode value => new BEncodedString (value.ToString ()),
                     CachePolicy value => new BEncodedString (value.ToString ()),
                     Uri value => new BEncodedString (value.OriginalString),
+                    FileCreationOptions value => new BEncodedString (value.ToString ()),
                     null => null,
                     Dictionary<string, IPEndPoint> value => FromIPAddressDictionary(value),
                     _ => throw new NotSupportedException ($"{property.Name} => type: ${property.PropertyType}"),