diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index ef94a768..1a701a72 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -259,24 +259,86 @@ public async Task DownloadAsync(CancellationToken cancellationToken) DownloadType downloadType = downloadOptions.Id.All(char.IsDigit) ? DownloadType.Video : DownloadType.Clip; - ChatRoot chatRoot = new() + var (chatRoot, connectionCount) = await InitChatRoot(downloadType); + var videoStart = chatRoot.video.start; + var videoEnd = chatRoot.video.end; + var videoId = chatRoot.video.id; + var videoDuration = videoEnd - videoStart; + + var downloadTasks = new List>>(connectionCount); + var percentages = new int[connectionCount]; + + double chunk = videoDuration / connectionCount; + for (int i = 0; i < connectionCount; i++) + { + int tc = i; + + var taskProgress = new Progress(percent => + { + percentages[tc] = Math.Clamp(percent, 0, 100); + + var reportPercent = percentages.Sum() / connectionCount; + _progress.ReportProgress(reportPercent); + }); + + double start = videoStart + chunk * i; + downloadTasks.Add(DownloadSection(start, start + chunk, videoId, taskProgress, downloadOptions.DownloadFormat, cancellationToken)); + } + + _progress.SetTemplateStatus("Downloading {0}%", 0); + await Task.WhenAll(downloadTasks); + + var sortedComments = new List(downloadTasks.Count); + foreach (var commentTask in downloadTasks) + { + sortedComments.AddRange(commentTask.Result); + } + + sortedComments.Sort(new CommentOffsetComparer()); + + chatRoot.comments = sortedComments.DistinctBy(x => x._id).ToList(); + + if (downloadOptions.EmbedData && (downloadOptions.DownloadFormat is ChatFormat.Json or ChatFormat.Html)) + { + await EmbedImages(chatRoot, cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (downloadOptions.DownloadFormat is ChatFormat.Json) + { + await BackfillUserInfo(chatRoot); + } + + _progress.SetStatus("Writing output file"); + switch (downloadOptions.DownloadFormat) + { + case ChatFormat.Json: + await ChatJson.SerializeAsync(outputFs, chatRoot, downloadOptions.Compression, cancellationToken); + break; + case ChatFormat.Html: + await ChatHtml.SerializeAsync(outputFs, outputFileInfo.FullName, chatRoot, _progress, downloadOptions.EmbedData, cancellationToken); + break; + case ChatFormat.Text: + await ChatText.SerializeAsync(outputFs, chatRoot, downloadOptions.TimeFormat); + break; + default: + throw new NotSupportedException($"{downloadOptions.DownloadFormat} is not a supported output format."); + } + } + + private async Task<(ChatRoot chatRoot, int connectionCount)> InitChatRoot(DownloadType downloadType) + { + var chatRoot = new ChatRoot { FileInfo = new ChatRootInfo { Version = ChatRootVersion.CurrentVersion, CreatedAt = DateTime.Now }, - streamer = new(), - video = new(), + streamer = new Streamer(), + video = new Video(), comments = new List() }; string videoId = downloadOptions.Id; - string videoTitle; - DateTime videoCreatedAt; - double videoStart = 0.0; - double videoEnd = 0.0; - double videoDuration = 0.0; - double videoTotalLength; - int viewCount; - string game; - int connectionCount = downloadOptions.DownloadThreads; + int connectionCount; if (downloadType == DownloadType.Video) { @@ -289,13 +351,14 @@ public async Task DownloadAsync(CancellationToken cancellationToken) chatRoot.streamer.name = videoInfoResponse.data.video.owner.displayName; chatRoot.streamer.id = int.Parse(videoInfoResponse.data.video.owner.id); chatRoot.video.description = videoInfoResponse.data.video.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(); - videoTitle = videoInfoResponse.data.video.title; - videoCreatedAt = videoInfoResponse.data.video.createdAt; - videoStart = downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : 0.0; - videoEnd = downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : videoInfoResponse.data.video.lengthSeconds; - videoTotalLength = videoInfoResponse.data.video.lengthSeconds; - viewCount = videoInfoResponse.data.video.viewCount; - game = videoInfoResponse.data.video.game?.displayName ?? "Unknown"; + chatRoot.video.title = videoInfoResponse.data.video.title; + chatRoot.video.created_at = videoInfoResponse.data.video.createdAt; + chatRoot.video.start = downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : 0.0; + chatRoot.video.end = downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : videoInfoResponse.data.video.lengthSeconds; + chatRoot.video.length = videoInfoResponse.data.video.lengthSeconds; + chatRoot.video.viewCount = videoInfoResponse.data.video.viewCount; + chatRoot.video.game = videoInfoResponse.data.video.game?.displayName ?? "Unknown"; + connectionCount = downloadOptions.DownloadThreads; GqlVideoChapterResponse videoChapterResponse = await TwitchHelper.GetOrGenerateVideoChapters(long.Parse(videoId), videoInfoResponse.data.video); chatRoot.video.chapters.EnsureCapacity(videoChapterResponse.data.video.moments.edges.Count); @@ -331,13 +394,13 @@ public async Task DownloadAsync(CancellationToken cancellationToken) downloadOptions.TrimEndingTime = downloadOptions.TrimBeginningTime + clipInfoResponse.data.clip.durationSeconds; chatRoot.streamer.name = clipInfoResponse.data.clip.broadcaster.displayName; chatRoot.streamer.id = int.Parse(clipInfoResponse.data.clip.broadcaster.id); - videoTitle = clipInfoResponse.data.clip.title; - videoCreatedAt = clipInfoResponse.data.clip.createdAt; - videoStart = (int)clipInfoResponse.data.clip.videoOffsetSeconds; - videoEnd = (int)clipInfoResponse.data.clip.videoOffsetSeconds + clipInfoResponse.data.clip.durationSeconds; - videoTotalLength = clipInfoResponse.data.clip.durationSeconds; - viewCount = clipInfoResponse.data.clip.viewCount; - game = clipInfoResponse.data.clip.game?.displayName ?? "Unknown"; + chatRoot.video.title = clipInfoResponse.data.clip.title; + chatRoot.video.created_at = clipInfoResponse.data.clip.createdAt; + chatRoot.video.start = (int)clipInfoResponse.data.clip.videoOffsetSeconds; + chatRoot.video.end = (int)clipInfoResponse.data.clip.videoOffsetSeconds + clipInfoResponse.data.clip.durationSeconds; + chatRoot.video.length = clipInfoResponse.data.clip.durationSeconds; + chatRoot.video.viewCount = clipInfoResponse.data.clip.viewCount; + chatRoot.video.game = clipInfoResponse.data.clip.game?.displayName ?? "Unknown"; connectionCount = 1; var clipChapter = TwitchHelper.GenerateClipChapter(clipInfoResponse.data.clip); @@ -357,184 +420,154 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } chatRoot.video.id = videoId; - chatRoot.video.title = videoTitle; - chatRoot.video.created_at = videoCreatedAt; - chatRoot.video.start = videoStart; - chatRoot.video.end = videoEnd; - chatRoot.video.length = videoTotalLength; - chatRoot.video.viewCount = viewCount; - chatRoot.video.game = game; - videoDuration = videoEnd - videoStart; - var downloadTasks = new List>>(connectionCount); - var percentages = new int[connectionCount]; + return (chatRoot, connectionCount); + } - double chunk = videoDuration / connectionCount; - for (int i = 0; i < connectionCount; i++) + private async Task EmbedImages(ChatRoot chatRoot, CancellationToken cancellationToken) + { + _progress.SetTemplateStatus("Downloading + Embedding Images {0}%", 0); + chatRoot.embeddedData = new EmbeddedData(); + + // This is the exact same process as in ChatUpdater.cs but not in a task oriented manner + // TODO: Combine this with ChatUpdater in a different file + List thirdPartyEmotes = await TwitchHelper.GetThirdPartyEmotes(chatRoot.comments, chatRoot.streamer.id, downloadOptions.TempFolder, _progress, bttv: downloadOptions.BttvEmotes, ffz: downloadOptions.FfzEmotes, stv: downloadOptions.StvEmotes, cancellationToken: cancellationToken); + _progress.ReportProgress(50 / 4); + List firstPartyEmotes = await TwitchHelper.GetEmotes(chatRoot.comments, downloadOptions.TempFolder, _progress, cancellationToken: cancellationToken); + _progress.ReportProgress(50 / 4 * 2); + List twitchBadges = await TwitchHelper.GetChatBadges(chatRoot.comments, chatRoot.streamer.id, downloadOptions.TempFolder, _progress, cancellationToken: cancellationToken); + _progress.ReportProgress(50 / 4 * 3); + List twitchBits = await TwitchHelper.GetBits(chatRoot.comments, downloadOptions.TempFolder, chatRoot.streamer.id.ToString(), _progress, cancellationToken: cancellationToken); + _progress.ReportProgress(50); + + var totalImageCount = thirdPartyEmotes.Count + firstPartyEmotes.Count + twitchBadges.Count + twitchBits.Count; + var imagesProcessed = 0; + + foreach (TwitchEmote emote in thirdPartyEmotes) { - int tc = i; - - var taskProgress = new Progress(percent => + var newEmote = new EmbedEmoteData { - percentages[tc] = Math.Clamp(percent, 0, 100); - - var reportPercent = percentages.Sum() / connectionCount; - _progress.ReportProgress(reportPercent); - }); + id = emote.Id, + imageScale = emote.ImageScale, + data = emote.ImageData, + name = emote.Name, + width = emote.Width / emote.ImageScale, + height = emote.Height / emote.ImageScale + }; - double start = videoStart + chunk * i; - downloadTasks.Add(DownloadSection(start, start + chunk, videoId, taskProgress, downloadOptions.DownloadFormat, cancellationToken)); + chatRoot.embeddedData.thirdParty.Add(newEmote); + _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); } - _progress.SetTemplateStatus("Downloading {0}%", 0); - await Task.WhenAll(downloadTasks); + cancellationToken.ThrowIfCancellationRequested(); - var sortedComments = new List(downloadTasks.Count); - foreach (var commentTask in downloadTasks) + foreach (TwitchEmote emote in firstPartyEmotes) { - sortedComments.AddRange(commentTask.Result); - } + var newEmote = new EmbedEmoteData + { + id = emote.Id, + imageScale = emote.ImageScale, + data = emote.ImageData, + width = emote.Width / emote.ImageScale, + height = emote.Height / emote.ImageScale, + isZeroWidth = emote.IsZeroWidth + }; - sortedComments.Sort(new CommentOffsetComparer()); + chatRoot.embeddedData.firstParty.Add(newEmote); + _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); + } - chatRoot.comments = sortedComments.DistinctBy(x => x._id).ToList(); + cancellationToken.ThrowIfCancellationRequested(); - if (downloadOptions.EmbedData && (downloadOptions.DownloadFormat is ChatFormat.Json or ChatFormat.Html)) + foreach (ChatBadge badge in twitchBadges) { - _progress.SetTemplateStatus("Downloading + Embedding Images {0}%", 0); - chatRoot.embeddedData = new EmbeddedData(); - - // This is the exact same process as in ChatUpdater.cs but not in a task oriented manner - // TODO: Combine this with ChatUpdater in a different file - List thirdPartyEmotes = await TwitchHelper.GetThirdPartyEmotes(chatRoot.comments, chatRoot.streamer.id, downloadOptions.TempFolder, _progress, bttv: downloadOptions.BttvEmotes, ffz: downloadOptions.FfzEmotes, stv: downloadOptions.StvEmotes, cancellationToken: cancellationToken); - _progress.ReportProgress(50 / 4); - List firstPartyEmotes = await TwitchHelper.GetEmotes(chatRoot.comments, downloadOptions.TempFolder, _progress, cancellationToken: cancellationToken); - _progress.ReportProgress(50 / 4 * 2); - List twitchBadges = await TwitchHelper.GetChatBadges(chatRoot.comments, chatRoot.streamer.id, downloadOptions.TempFolder, _progress, cancellationToken: cancellationToken); - _progress.ReportProgress(50 / 4 * 3); - List twitchBits = await TwitchHelper.GetBits(chatRoot.comments, downloadOptions.TempFolder, chatRoot.streamer.id.ToString(), cancellationToken: cancellationToken); - _progress.ReportProgress(50); - - var totalImageCount = thirdPartyEmotes.Count + firstPartyEmotes.Count + twitchBadges.Count + twitchBits.Count; - var imagesProcessed = 0; - - foreach (TwitchEmote emote in thirdPartyEmotes) + var newBadge = new EmbedChatBadge { - EmbedEmoteData newEmote = new EmbedEmoteData(); - newEmote.id = emote.Id; - newEmote.imageScale = emote.ImageScale; - newEmote.data = emote.ImageData; - newEmote.name = emote.Name; - newEmote.width = emote.Width / emote.ImageScale; - newEmote.height = emote.Height / emote.ImageScale; - chatRoot.embeddedData.thirdParty.Add(newEmote); - _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); - } - - cancellationToken.ThrowIfCancellationRequested(); + name = badge.Name, + versions = badge.VersionsData + }; - foreach (TwitchEmote emote in firstPartyEmotes) - { - EmbedEmoteData newEmote = new EmbedEmoteData(); - newEmote.id = emote.Id; - newEmote.imageScale = emote.ImageScale; - newEmote.data = emote.ImageData; - newEmote.width = emote.Width / emote.ImageScale; - newEmote.height = emote.Height / emote.ImageScale; - newEmote.isZeroWidth = emote.IsZeroWidth; - chatRoot.embeddedData.firstParty.Add(newEmote); - _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); - } + chatRoot.embeddedData.twitchBadges.Add(newBadge); + _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); + } - cancellationToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); - foreach (ChatBadge badge in twitchBadges) + foreach (CheerEmote bit in twitchBits) + { + var newBit = new EmbedCheerEmote { - EmbedChatBadge newBadge = new EmbedChatBadge(); - newBadge.name = badge.Name; - newBadge.versions = badge.VersionsData; - chatRoot.embeddedData.twitchBadges.Add(newBadge); - _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); - } - - cancellationToken.ThrowIfCancellationRequested(); + prefix = bit.prefix, + tierList = new Dictionary() + }; - foreach (CheerEmote bit in twitchBits) + foreach (KeyValuePair emotePair in bit.tierList) { - EmbedCheerEmote newBit = new EmbedCheerEmote(); - newBit.prefix = bit.prefix; - newBit.tierList = new Dictionary(); - foreach (KeyValuePair emotePair in bit.tierList) + EmbedEmoteData newEmote = new EmbedEmoteData { - EmbedEmoteData newEmote = new EmbedEmoteData(); - newEmote.id = emotePair.Value.Id; - newEmote.imageScale = emotePair.Value.ImageScale; - newEmote.data = emotePair.Value.ImageData; - newEmote.name = emotePair.Value.Name; - newEmote.width = emotePair.Value.Width / emotePair.Value.ImageScale; - newEmote.height = emotePair.Value.Height / emotePair.Value.ImageScale; - newBit.tierList.Add(emotePair.Key, newEmote); - } - chatRoot.embeddedData.twitchBits.Add(newBit); - _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); + id = emotePair.Value.Id, + imageScale = emotePair.Value.ImageScale, + data = emotePair.Value.ImageData, + name = emotePair.Value.Name, + width = emotePair.Value.Width / emotePair.Value.ImageScale, + height = emotePair.Value.Height / emotePair.Value.ImageScale + }; + newBit.tierList.Add(emotePair.Key, newEmote); } + + chatRoot.embeddedData.twitchBits.Add(newBit); + _progress.ReportProgress(++imagesProcessed * 100 / totalImageCount + 50); } + } - cancellationToken.ThrowIfCancellationRequested(); + private async Task BackfillUserInfo(ChatRoot chatRoot) + { + // Best effort, but if we fail oh well + _progress.SetTemplateStatus("Backfilling Commenter Info {0}", 0); - if (downloadOptions.DownloadFormat is ChatFormat.Json) + var userIds = chatRoot.comments.Select(x => x.commenter._id).Distinct().ToArray(); + var userInfo = new Dictionary(userIds.Length); + + var failedInfo = false; + const int BATCH_SIZE = 100; + for (var i = 0; i < userIds.Length; i += BATCH_SIZE) { - //Best effort, but if we fail oh well - _progress.SetStatus("Backfilling commenter info"); - List userList = chatRoot.comments.DistinctBy(x => x.commenter._id).Select(x => x.commenter._id).ToList(); - Dictionary userInfo = new Dictionary(); - int batchSize = 100; - bool failedInfo = false; - for (int i = 0; i <= userList.Count / batchSize; i++) + try { - try + var userSubset = userIds.Skip(i).Take(BATCH_SIZE); + + GqlUserInfoResponse userInfoResponse = await TwitchHelper.GetUserInfo(userSubset); + foreach (var user in userInfoResponse.data.users) { - List userSubset = userList.Skip(i * batchSize).Take(batchSize).ToList(); - GqlUserInfoResponse userInfoResponse = await TwitchHelper.GetUserInfo(userSubset); - foreach (var user in userInfoResponse.data.users) - { - userInfo[user.id] = user; - } + userInfo[user.id] = user; } - catch { failedInfo = true; } - } - if (failedInfo) - { - _progress.LogInfo("Failed to backfill some commenter info"); + var percent = (i + BATCH_SIZE) * 100f / userIds.Length; + _progress.ReportProgress((int)percent); } - - foreach (var comment in chatRoot.comments) + catch (Exception e) { - if (userInfo.TryGetValue(comment.commenter._id, out var user)) - { - comment.commenter.updated_at = user.updatedAt; - comment.commenter.created_at = user.createdAt; - comment.commenter.bio = user.description; - comment.commenter.logo = user.profileImageURL; - } + _progress.LogVerbose($"An error occurred while backfilling commenters {i}-{i + BATCH_SIZE}: {e.Message}"); + failedInfo = true; } } - _progress.SetStatus("Writing output file"); - switch (downloadOptions.DownloadFormat) + _progress.ReportProgress(100); + + if (failedInfo) { - case ChatFormat.Json: - await ChatJson.SerializeAsync(outputFs, chatRoot, downloadOptions.Compression, cancellationToken); - break; - case ChatFormat.Html: - await ChatHtml.SerializeAsync(outputFs, outputFileInfo.FullName, chatRoot, _progress, downloadOptions.EmbedData, cancellationToken); - break; - case ChatFormat.Text: - await ChatText.SerializeAsync(outputFs, chatRoot, downloadOptions.TimeFormat); - break; - default: - throw new NotSupportedException($"{downloadOptions.DownloadFormat} is not a supported output format."); + _progress.LogInfo("Failed to backfill some commenter info"); + } + + foreach (var comment in chatRoot.comments) + { + if (userInfo.TryGetValue(comment.commenter._id, out var user)) + { + comment.commenter.updated_at = user.updatedAt; + comment.commenter.created_at = user.createdAt; + comment.commenter.bio = user.description; + comment.commenter.logo = user.profileImageURL; + } } } } diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index 62bb68f6..6e04f6e9 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -1722,7 +1722,7 @@ private async Task> GetScaledThirdEmotes(CancellationToken can private async Task> GetScaledBits(CancellationToken cancellationToken) { - var cheerTask = await TwitchHelper.GetBits(chatRoot.comments, renderOptions.TempFolder, chatRoot.streamer.id.ToString(), chatRoot.embeddedData, renderOptions.Offline, cancellationToken); + var cheerTask = await TwitchHelper.GetBits(chatRoot.comments, renderOptions.TempFolder, chatRoot.streamer.id.ToString(), _progress, chatRoot.embeddedData, renderOptions.Offline, cancellationToken); foreach (var cheer in cheerTask) { diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index d2587d47..324b5a8f 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -306,7 +306,7 @@ private async Task ChatBadgeTask(CancellationToken cancellationToken = default) private async Task BitTask(CancellationToken cancellationToken = default) { - List bitList = await TwitchHelper.GetBits(chatRoot.comments, _updateOptions.TempFolder, chatRoot.streamer.id.ToString(), _updateOptions.ReplaceEmbeds ? null : chatRoot.embeddedData, cancellationToken: cancellationToken); + List bitList = await TwitchHelper.GetBits(chatRoot.comments, _updateOptions.TempFolder, chatRoot.streamer.id.ToString(), _progress, _updateOptions.ReplaceEmbeds ? null : chatRoot.embeddedData, cancellationToken: cancellationToken); int inputCount = chatRoot.embeddedData.twitchBits.Count; chatRoot.embeddedData.twitchBits = new List(); diff --git a/TwitchDownloaderCore/Tools/HighlightIcons.cs b/TwitchDownloaderCore/Tools/HighlightIcons.cs index aee70c39..1a986769 100644 --- a/TwitchDownloaderCore/Tools/HighlightIcons.cs +++ b/TwitchDownloaderCore/Tools/HighlightIcons.cs @@ -56,7 +56,7 @@ public sealed class HighlightIcons : IDisposable private SKImage _watchStreakIcon; private SKImage _charityDonationIcon; - private readonly string _cachePath; + private readonly DirectoryInfo _cacheDir; private readonly SKColor _purple; private readonly bool _offline; private readonly double _fontSize; @@ -66,7 +66,7 @@ public sealed class HighlightIcons : IDisposable public HighlightIcons(ChatRenderOptions renderOptions, SKColor iconPurple, SKPaint outlinePaint) { - _cachePath = Path.Combine(renderOptions.TempFolder, "icons"); + _cacheDir = new DirectoryInfo(Path.Combine(renderOptions.TempFolder, "icons")); _purple = iconPurple; _offline = renderOptions.Offline; _fontSize = renderOptions.FontSize; @@ -184,7 +184,7 @@ private SKImage GenerateGiftedManyIcon() return SKImage.FromBitmap(offlineBitmap); } - var taskIconBytes = TwitchHelper.GetImage(_cachePath, GIFTED_MANY_ICON_URL, "gift-illus", "3", "png"); + var taskIconBytes = TwitchHelper.GetImage(_cacheDir, GIFTED_MANY_ICON_URL, "gift-illus", 3, "png", StubTaskProgress.Instance); taskIconBytes.Wait(); using var ms = new MemoryStream(taskIconBytes.Result); // Illustration is 72x72 using var codec = SKCodec.Create(ms); diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs index 06b1d5e4..7ec093d8 100644 --- a/TwitchDownloaderCore/TwitchHelper.cs +++ b/TwitchDownloaderCore/TwitchHelper.cs @@ -1,7 +1,6 @@ using SkiaSharp; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Compression; using System.Linq; @@ -9,6 +8,7 @@ using System.Net.Http; using System.Net.Http.Json; using System.Runtime.InteropServices; +using System.Security; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -389,9 +389,9 @@ public static async Task> GetThirdPartyEmotes(List co return returnList; } - string bttvFolder = Path.Combine(cacheFolder, "bttv"); - string ffzFolder = Path.Combine(cacheFolder, "ffz"); - string stvFolder = Path.Combine(cacheFolder, "stv"); + DirectoryInfo bttvFolder = new DirectoryInfo(Path.Combine(cacheFolder, "bttv")); + DirectoryInfo ffzFolder = new DirectoryInfo(Path.Combine(cacheFolder, "ffz")); + DirectoryInfo stvFolder = new DirectoryInfo(Path.Combine(cacheFolder, "stv")); EmoteResponse emoteDataResponse = await GetThirdPartyEmotesMetadata(streamerId, bttv, ffz, stv, allowUnlistedEmotes, logger, cancellationToken); @@ -434,10 +434,10 @@ public static async Task> GetThirdPartyEmotes(List co return returnList; static async Task FetchEmoteImages(IReadOnlyCollection comments, IEnumerable emoteResponse, ICollection returnList, - ICollection alreadyAdded, string cacheFolder, ITaskLogger logger, CancellationToken cancellationToken) + ICollection alreadyAdded, DirectoryInfo cacheFolder, ITaskLogger logger, CancellationToken cancellationToken) { - if (!Directory.Exists(cacheFolder)) - CreateDirectory(cacheFolder); + if (!cacheFolder.Exists) + cacheFolder = CreateDirectory(cacheFolder.FullName); IEnumerable emoteResponseQuery; if (comments.Count == 0) @@ -457,7 +457,7 @@ where comments.Any(comment => Regex.IsMatch(comment.message.body, pattern)) { try { - var imageData = await GetImage(cacheFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType, cancellationToken); + var imageData = await GetImage(cacheFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, 2, emote.ImageType, logger, cancellationToken); var newEmote = new TwitchEmote(imageData, EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); newEmote.IsZeroWidth = emote.IsZeroWidth; @@ -478,9 +478,9 @@ public static async Task> GetEmotes(List comments, st List alreadyAdded = new List(); List failedEmotes = new List(); - string emoteFolder = Path.Combine(cacheFolder, "emotes"); - if (!Directory.Exists(emoteFolder)) - TwitchHelper.CreateDirectory(emoteFolder); + DirectoryInfo emoteFolder = new DirectoryInfo(Path.Combine(cacheFolder, "emotes")); + if (!emoteFolder.Exists) + emoteFolder = CreateDirectory(emoteFolder.FullName); // Load our embedded emotes if (embeddedData?.firstParty != null) @@ -518,7 +518,7 @@ public static async Task> GetEmotes(List comments, st { try { - byte[] bytes = await GetImage(emoteFolder, $"https://static-cdn.jtvnw.net/emoticons/v2/{id}/default/dark/2.0", id, "2", "png", cancellationToken); + byte[] bytes = await GetImage(emoteFolder, $"https://static-cdn.jtvnw.net/emoticons/v2/{id}/default/dark/2.0", id, 2, "png", logger, cancellationToken); TwitchEmote newEmote = new TwitchEmote(bytes, EmoteProvider.FirstParty, 2, id, id); alreadyAdded.Add(id); returnList.Add(newEmote); @@ -638,9 +638,9 @@ public static async Task> GetChatBadges(List comments, List badgesData = await GetChatBadgesData(comments, streamerId, cancellationToken); - string badgeFolder = Path.Combine(cacheFolder, "badges"); - if (!Directory.Exists(badgeFolder)) - TwitchHelper.CreateDirectory(badgeFolder); + DirectoryInfo badgeFolder = new DirectoryInfo(Path.Combine(cacheFolder, "badges")); + if (!badgeFolder.Exists) + badgeFolder = CreateDirectory(badgeFolder.FullName); foreach(var badge in badgesData) { @@ -654,7 +654,7 @@ public static async Task> GetChatBadges(List comments, foreach (var (version, data) in badge.versions) { string id = data.url.Split('/')[^2]; - byte[] bytes = await GetImage(badgeFolder, data.url, id, "2", "png", cancellationToken); + byte[] bytes = await GetImage(badgeFolder, data.url, id, 2, "png", logger, cancellationToken); versions.Add(version, new ChatBadgeData { title = data.title, @@ -756,7 +756,7 @@ public static async Task> GetEmojis(string cacheFol return returnCache; } - public static async Task> GetBits(List comments, string cacheFolder, string channelId = "", EmbeddedData embeddedData = null, bool offline = false, CancellationToken cancellationToken = default) + public static async Task> GetBits(List comments, string cacheFolder, string channelId, ITaskLogger logger, EmbeddedData embeddedData = null, bool offline = false, CancellationToken cancellationToken = default) { List returnList = new List(); List alreadyAdded = new List(); @@ -768,15 +768,23 @@ public static async Task> GetBits(List comments, strin { cancellationToken.ThrowIfCancellationRequested(); - List> tierList = new List>(); - CheerEmote newEmote = new CheerEmote() { prefix = data.prefix, tierList = tierList }; - foreach (KeyValuePair tier in data.tierList) + try { - TwitchEmote tierEmote = new TwitchEmote(tier.Value.data, EmoteProvider.FirstParty, tier.Value.imageScale, tier.Value.id, tier.Value.name); - tierList.Add(new KeyValuePair(tier.Key, tierEmote)); + List> tierList = new List>(); + CheerEmote newEmote = new CheerEmote() { prefix = data.prefix, tierList = tierList }; + foreach (KeyValuePair tier in data.tierList) + { + TwitchEmote tierEmote = new TwitchEmote(tier.Value.data, EmoteProvider.FirstParty, tier.Value.imageScale, tier.Value.id, tier.Value.name); + tierList.Add(new KeyValuePair(tier.Key, tierEmote)); + } + + returnList.Add(newEmote); + alreadyAdded.Add(data.prefix); + } + catch (Exception e) + { + logger.LogVerbose($"An exception occurred while loading embedded cheermote '{data.prefix}': {e.Message}."); } - returnList.Add(newEmote); - alreadyAdded.Add(data.prefix); } } @@ -797,9 +805,9 @@ public static async Task> GetBits(List comments, strin cheerResponseMessage.EnsureSuccessStatusCode(); var cheerResponse = await cheerResponseMessage.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); - string bitFolder = Path.Combine(cacheFolder, "bits"); - if (!Directory.Exists(bitFolder)) - TwitchHelper.CreateDirectory(bitFolder); + DirectoryInfo bitFolder = new DirectoryInfo(Path.Combine(cacheFolder, "bits")); + if (!bitFolder.Exists) + bitFolder = CreateDirectory(bitFolder.FullName); if (cheerResponse?.data != null) { @@ -841,7 +849,8 @@ where comments { int minBits = tier.bits; string url = templateURL.Replace("PREFIX", node.prefix.ToLower()).Replace("BACKGROUND", "dark").Replace("ANIMATION", "animated").Replace("TIER", tier.bits.ToString()).Replace("SCALE.EXTENSION", "2.gif"); - TwitchEmote emote = new TwitchEmote(await GetImage(bitFolder, url, node.id + tier.bits, "2", "gif", cancellationToken), EmoteProvider.FirstParty, 2, prefix + minBits, prefix + minBits); + var bytes = await GetImage(bitFolder, url, node.id + tier.bits, 2, "gif", logger, cancellationToken); + TwitchEmote emote = new TwitchEmote(bytes, EmoteProvider.FirstParty, 2, prefix + minBits, prefix + minBits); tierList.Add(new KeyValuePair(minBits, emote)); } returnList.Add(newEmote); @@ -873,7 +882,10 @@ public static FileInfo ClaimFile(string path, Func fileAlrea throw new FileNotFoundException("No destination file was provided, aborting."); } - logger.LogVerbose($"{path} will be renamed to {fileInfo.FullName}."); + if (path != fileInfo.FullName) + { + logger.LogInfo($"'{path}' will be renamed to '{fileInfo.FullName}'"); + } } } @@ -986,7 +998,7 @@ public static async Task GetStreamerName(int id) catch { return ""; } } - public static async Task GetUserInfo(List idList) + public static async Task GetUserInfo(IEnumerable idList) { var request = new HttpRequestMessage() { @@ -1000,65 +1012,64 @@ public static async Task GetUserInfo(List idList) return await response.Content.ReadFromJsonAsync(); } - public static async Task GetImage(string cachePath, string imageUrl, string imageId, string imageScale, string imageType, CancellationToken cancellationToken = new()) + public static async Task GetImage(DirectoryInfo cacheDir, string imageUrl, string imageId, int imageScale, string imageType, ITaskLogger logger, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - byte[] imageBytes = null; + cacheDir.Refresh(); + if (!cacheDir.Exists) + { + CreateDirectory(cacheDir.FullName); + cacheDir.Refresh(); + } + + byte[] imageBytes; - if (!Directory.Exists(cachePath)) - CreateDirectory(cachePath); + var filePath = Path.Combine(cacheDir.FullName, $"{imageId}_{imageScale}.{imageType}"); + var file = new FileInfo(filePath); - string filePath = Path.Combine(cachePath!, imageId + "_" + imageScale + "." + imageType); - if (File.Exists(filePath)) + if (file.Exists) { try { - await using FileStream stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); - byte[] bytes = new byte[stream.Length]; - stream.Seek(0, SeekOrigin.Begin); - _ = await stream.ReadAsync(bytes, cancellationToken); + await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + imageBytes = new byte[fs.Length]; + _ = await fs.ReadAsync(imageBytes, cancellationToken); - //Check if image file is not corrupt - if (bytes.Length > 0) + if (imageBytes.Length > 0) { - using SKImage image = SKImage.FromEncodedData(bytes); - if (image != null) - { - imageBytes = bytes; - } - else + using var ms = new MemoryStream(imageBytes); + using var codec = SKCodec.Create(ms, out var result); + + if (codec is not null) { - //Try to delete the corrupted image - try - { - await stream.DisposeAsync(); - File.Delete(filePath); - } - catch { } + return imageBytes; } + + logger.LogVerbose($"Failed to decode {imageId} from cache: {result}"); } + + // Delete the corrupted image + file.Delete(); } - catch (IOException) + catch (Exception e) when (e is IOException or SecurityException) { - //File being written to by parallel process? Maybe. Can just fallback to HTTP request. + // File being written to by parallel process? Maybe. Can just fallback to HTTP request. + logger.LogVerbose($"Failed to read from or delete {file.Name}: {e.Message}"); } } - // If fetching from cache failed - if (imageBytes != null) - return imageBytes; - - // Fallback to HTTP request imageBytes = await httpClient.GetByteArrayAsync(imageUrl, cancellationToken); - //Let's save this image to the cache try { - await using var stream = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); - await stream.WriteAsync(imageBytes, cancellationToken); + await using var fs = file.Open(FileMode.Create, FileAccess.Write, FileShare.Read); + await fs.WriteAsync(imageBytes, cancellationToken); + } + catch (Exception e) + { + logger.LogVerbose($"Failed to open or write to {file.Name}: {e.Message}"); } - catch { } return imageBytes; }