diff --git a/TwitchDownloaderCore/Tools/DownloadTools.cs b/TwitchDownloaderCore/Tools/DownloadTools.cs
index 90ed883d..0997872f 100644
--- a/TwitchDownloaderCore/Tools/DownloadTools.cs
+++ b/TwitchDownloaderCore/Tools/DownloadTools.cs
@@ -18,8 +18,9 @@ public static class DownloadTools
/// The maximum download speed in kibibytes per second, or -1 for no maximum.
/// Logger.
/// A containing a to cancel the operation.
+ /// The expected length of the downloaded file, or -1 if the content length header is not present.
/// The may be canceled by this method.
- public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
+ public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
@@ -74,6 +75,8 @@ public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, strin
// Reset the cts timer so it can be reused for the next download on this thread.
// Is there a friendlier way to do this? Yes. Does it involve creating and destroying 4,000 CancellationTokenSources that are almost never cancelled? Also Yes.
cancellationTokenSource?.CancelAfter(TimeSpan.FromMilliseconds(uint.MaxValue - 1));
+
+ return response.Content.Headers.ContentLength ?? -1;
}
diff --git a/TwitchDownloaderCore/Tools/VideoDownloadThread.cs b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs
index e4162096..527d5708 100644
--- a/TwitchDownloaderCore/Tools/VideoDownloadThread.cs
+++ b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs
@@ -77,8 +77,7 @@ private void ExecuteDownloadThread()
throw;
}
- const int A_PRIME_NUMBER = 71;
- Thread.Sleep(A_PRIME_NUMBER);
+ Thread.Sleep(Random.Shared.Next(50, 150));
}
}
@@ -97,6 +96,7 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke
var tryUnmute = VodAge < TimeSpan.FromHours(24);
var errorCount = 0;
var timeoutCount = 0;
+ var lengthFailureCount = 0;
while (true)
{
cancellationTokenSource.Token.ThrowIfCancellationRequested();
@@ -104,14 +104,40 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke
try
{
var partFile = Path.Combine(_cacheFolder, DownloadTools.RemoveQueryString(videoPartName));
+ long expectedLength;
if (tryUnmute && videoPartName.Contains("-muted"))
{
var unmutedPartName = videoPartName.Replace("-muted", "");
- await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, unmutedPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
+ expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, unmutedPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
}
else
{
- await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, videoPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
+ expectedLength = await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, videoPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
+ }
+
+ if (expectedLength is not -1)
+ {
+ // I would love to compare hashes here but unfortunately Twitch doesn't give us a ContentMD5 header
+ var actualLength = new FileInfo(partFile).Length;
+ if (actualLength != expectedLength)
+ {
+ const int MAX_RETRIES = 1;
+
+ _logger.LogVerbose($"{partFile} failed to verify: expected {expectedLength:N0}B, got {actualLength:N0}B.");
+ if (++lengthFailureCount > MAX_RETRIES)
+ {
+ throw new Exception($"Failed to download {partFile}: expected {expectedLength:N0}B, got {actualLength:N0}B.");
+ }
+
+ await Delay(1_000, cancellationTokenSource.Token);
+ continue;
+ }
+
+ const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf
+ if (expectedLength % TS_PACKET_LENGTH != 0)
+ {
+ _logger.LogVerbose($"{partFile} contains malformed packets and may cause encoding issues.");
+ }
}
return;
@@ -121,7 +147,7 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke
_logger.LogVerbose($"Received {ex.StatusCode}: {ex.StatusCode} when trying to unmute {videoPartName}. Disabling {nameof(tryUnmute)}.");
tryUnmute = false;
- await Task.Delay(100, cancellationTokenSource.Token);
+ await Delay(100, cancellationTokenSource.Token);
}
catch (HttpRequestException ex)
{
@@ -133,7 +159,7 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke
throw new HttpRequestException($"Video part {videoPartName} failed after {MAX_RETRIES} retries");
}
- await Task.Delay(1_000 * errorCount, cancellationTokenSource.Token);
+ await Delay(1_000 * errorCount, cancellationTokenSource.Token);
}
catch (TaskCanceledException ex) when (ex.Message.Contains("HttpClient.Timeout"))
{
@@ -145,9 +171,15 @@ private async Task DownloadVideoPartAsync(string videoPartName, CancellationToke
throw new HttpRequestException($"Video part {videoPartName} timed out {MAX_RETRIES} times");
}
- await Task.Delay(5_000 * timeoutCount, cancellationTokenSource.Token);
+ await Delay(5_000 * timeoutCount, cancellationTokenSource.Token);
}
}
}
+
+ private static Task Delay(int millis, CancellationToken cancellationToken)
+ {
+ var jitteredMillis = millis + Random.Shared.Next(-200, 200);
+ return Task.Delay(Math.Max(millis / 2, jitteredMillis), cancellationToken);
+ }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs
index 232186fa..d340ea35 100644
--- a/TwitchDownloaderCore/VideoDownloader.cs
+++ b/TwitchDownloaderCore/VideoDownloader.cs
@@ -298,7 +298,8 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang
foreach (var part in playlist.Take(videoListCrop))
{
var filePath = Path.Combine(downloadFolder, DownloadTools.RemoveQueryString(part.Path));
- if (!VerifyVideoPart(filePath))
+ var fi = new FileInfo(filePath);
+ if (!fi.Exists || fi.Length == 0)
{
failedParts.Add(part);
}
@@ -312,17 +313,11 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang
if (failedParts.Count != 0)
{
- if (playlist.Count == 1)
- {
- // The video is only 1 part, it probably won't be a complete file.
- return;
- }
-
if (partCount > 20 && failedParts.Count >= partCount * 0.95)
{
- // 19/20 parts failed to verify. Either the VOD is heavily corrupted or something went horribly wrong.
+ // 19/20 parts are missing or empty, something went horribly wrong.
// TODO: Somehow let the user bypass this. Maybe with callbacks?
- throw new Exception($"Too many parts are corrupted or missing ({failedParts.Count}/{partCount}), aborting.");
+ throw new Exception($"Too many parts are missing ({failedParts.Count}/{partCount}), aborting.");
}
_progress.LogInfo($"The following parts will be redownloaded: {string.Join(", ", failedParts)}");
@@ -330,25 +325,6 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang
}
}
- private static bool VerifyVideoPart(string filePath)
- {
- const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf
-
- if (!File.Exists(filePath))
- {
- return false;
- }
-
- using var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
- var fileLength = fs.Length;
- if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0)
- {
- return false;
- }
-
- return true;
- }
-
private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, decimal endOffset, TimeSpan videoLength)
{
using var process = new Process