From ef143bb3e5eb1114644ecdac0c8a21fd543d0c12 Mon Sep 17 00:00:00 2001 From: Patrick Geneva Date: Sun, 20 Nov 2022 16:58:54 -0500 Subject: [PATCH] Ability to embed twitch badges and bit emotes (#437) * add ability to embed twitch badges and bit emotes * render using the cached ones! * add --offline argument for CLI * script to update old chat json files * Renamed 'emotes' json property to 'embededData', renamed '--embed-emotes' to '--embed-images', update README, complete some TODOs, typos * Typos, added some comments * GetImage can fail, put in try catch * use status codes in the catch statements Co-authored-by: ScrubN <72096833+ScrubN@users.noreply.github.com> Co-authored-by: Lewis Pardo --- .../Modes/Arguments/ChatDownloadArgs.cs | 10 +- .../Arguments/ChatDownloadUpdaterArgs.cs | 33 +++ .../Modes/Arguments/ChatRenderArgs.cs | 3 + TwitchDownloaderCLI/Modes/DownloadChat.cs | 2 +- .../Modes/DownloadChatUpdater.cs | 170 +++++++++++++++ TwitchDownloaderCLI/Modes/RenderChat.cs | 3 +- TwitchDownloaderCLI/Program.cs | 11 +- TwitchDownloaderCLI/README.md | 45 +++- TwitchDownloaderCLI/Tools/PreParseArgs.cs | 54 +++-- TwitchDownloaderCore/ChatDownloader.cs | 70 +++++-- TwitchDownloaderCore/ChatRenderer.cs | 61 ++++-- .../Options/ChatDownloadOptions.cs | 2 +- .../Options/ChatRenderOptions.cs | 1 + TwitchDownloaderCore/TwitchHelper.cs | 195 ++++++++++++------ .../TwitchObjects/ChatBadge.cs | 21 +- .../TwitchObjects/ChatRoot.cs | 21 +- TwitchDownloaderWPF/PageChatDownload.xaml.cs | 2 +- TwitchDownloaderWPF/PageChatRender.xaml.cs | 3 +- .../WindowQueueOptions.xaml.cs | 6 +- 19 files changed, 573 insertions(+), 140 deletions(-) create mode 100644 TwitchDownloaderCLI/Modes/Arguments/ChatDownloadUpdaterArgs.cs create mode 100644 TwitchDownloaderCLI/Modes/DownloadChatUpdater.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs index 1cc50861..70cf84fc 100644 --- a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs +++ b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs @@ -19,16 +19,16 @@ public class ChatDownloadArgs [Option('e', "ending", HelpText = "Time in seconds to crop ending.")] public int CropEndingTime { get; set; } - [Option('E', "embed-emotes", Default = false, HelpText = "Embed emotes into the chat download.")] - public bool EmbedEmotes { get; set; } + [Option('E', "embed-images", Default = false, HelpText = "Embed first party emotes, badges, and cheermotes into the chat download for offline rendering.")] + public bool EmbedData { get; set; } - [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download. Requires -E / --embed-emotes!")] + [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download. Requires -E / --embed-images!")] public bool? BttvEmotes { get; set; } - [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download. Requires -E / --embed-emotes!")] + [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download. Requires -E / --embed-images!")] public bool? FfzEmotes { get; set; } - [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download. Requires -E / --embed-emotes!")] + [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download. Requires -E / --embed-images!")] public bool? StvEmotes { get; set; } [Option("timestamp", Default = false, HelpText = "Enable timestamps for .txt chat downloads.")] diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadUpdaterArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadUpdaterArgs.cs new file mode 100644 index 00000000..b5466abf --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadUpdaterArgs.cs @@ -0,0 +1,33 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + + [Verb("chatupdate", HelpText = "Updates the embeded emotes, badges, and bits of a chat download.")] + public class ChatDownloadUpdaterArgs + { + [Option('i', "input", Required = true, HelpText = "Path to input file. Valid extensions are json")] + public string InputFile { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file. Extension should match the input.")] + public string OutputFile { get; set; } + + [Option('E', "embed-missing", Default = true, HelpText = "Embed missing emotes, badges, and bits. Already embedded images will be untouched")] + public bool EmbedMissing { get; set; } + + [Option('U', "update-old", Default = false, HelpText = "Update old emotes, badges, and bits to the current. All embedded images will be overwritten")] + public bool UpdateOldEmbeds { get; set; } + + [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download.")] + public bool BttvEmotes { get; set; } + + [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download.")] + public bool FfzEmotes { get; set; } + + [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download.")] + public bool StvEmotes { get; set; } + + [Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")] + public string TempFolder { get; set; } + } +} diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs index e0926eda..b83694d8 100644 --- a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs +++ b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs @@ -81,6 +81,9 @@ public class ChatRenderArgs [Option("badge-filter", Default = 0, HelpText = "Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, 6 = no broadcaster or moderator badges.\r\nKey: Other = 1, Broadcaster = 2, Moderator = 4, VIP = 8, Subscriber = 16, Predictions = 32, NoAudio/NoVideo = 64, PrimeGaming = 128")] public int BadgeFilterMask { get; set; } + [Option("offline", Default = false, HelpText = "Render completely offline, using only resources embedded emotes, badges, and bits in the input json.")] + public bool Offline { get; set; } + [Option("ffmpeg-path", HelpText = "Path to ffmpeg executable.")] public string FfmpegPath { get; set; } diff --git a/TwitchDownloaderCLI/Modes/DownloadChat.cs b/TwitchDownloaderCLI/Modes/DownloadChat.cs index cfe2e138..47efc3f0 100644 --- a/TwitchDownloaderCLI/Modes/DownloadChat.cs +++ b/TwitchDownloaderCLI/Modes/DownloadChat.cs @@ -33,7 +33,7 @@ internal static void Download(ChatDownloadArgs inputOptions) CropEnding = inputOptions.CropEndingTime > 0.0, CropEndingTime = inputOptions.CropEndingTime, Timestamp = inputOptions.Timestamp, - EmbedEmotes = inputOptions.EmbedEmotes, + EmbedData = inputOptions.EmbedData, Filename = inputOptions.OutputFile, TimeFormat = inputOptions.TimeFormat, ConnectionCount = inputOptions.ChatConnections, diff --git a/TwitchDownloaderCLI/Modes/DownloadChatUpdater.cs b/TwitchDownloaderCLI/Modes/DownloadChatUpdater.cs new file mode 100644 index 00000000..fd459997 --- /dev/null +++ b/TwitchDownloaderCLI/Modes/DownloadChatUpdater.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using TwitchDownloaderCLI.Modes.Arguments; +using TwitchDownloaderCore; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.TwitchObjects; + +namespace TwitchDownloaderCLI.Modes +{ + internal class DownloadChatUpdater + { + internal static void Update(ChatDownloadUpdaterArgs inputOptions) + { + DownloadFormat inFormat = Path.GetExtension(inputOptions.InputFile)!.ToLower() switch + { + ".json" => DownloadFormat.Json, + ".html" => DownloadFormat.Html, + ".htm" => DownloadFormat.Html, + _ => DownloadFormat.Text + }; + DownloadFormat outFormat = Path.GetExtension(inputOptions.OutputFile)!.ToLower() switch + { + ".json" => DownloadFormat.Json, + ".html" => DownloadFormat.Html, + ".htm" => DownloadFormat.Html, + _ => DownloadFormat.Text + }; + // Check that both input and output are json + if (inFormat != DownloadFormat.Json || outFormat != DownloadFormat.Json) + { + Console.WriteLine("[ERROR] - {0} format must be be json!", inFormat != DownloadFormat.Json ? "Input" : "Output"); + Environment.Exit(1); + } + if (!File.Exists(inputOptions.InputFile)) + { + Console.WriteLine("[ERROR] - Input file does not exist!"); + Environment.Exit(1); + } + if (!inputOptions.EmbedMissing && !inputOptions.UpdateOldEmbeds) + { + Console.WriteLine("[ERROR] - Please enable either EmbedMissingEmotes or UpdateOldEmotes"); + Environment.Exit(1); + } + + // Read in the old input file + ChatRoot chatRoot = Task.Run(() => ChatRenderer.ParseJsonStatic(inputOptions.InputFile)).Result; + if (chatRoot.streamer == null) + { + chatRoot.streamer = new Streamer(); + chatRoot.streamer.id = int.Parse(chatRoot.comments.First().channel_id); + chatRoot.streamer.name = Task.Run(() => TwitchHelper.GetStreamerName(chatRoot.streamer.id)).Result; + } + if (chatRoot.embeddedData == null) + { + chatRoot.embeddedData = new EmbeddedData(); + } + + string cacheFolder = Path.Combine(string.IsNullOrWhiteSpace(inputOptions.TempFolder) ? Path.GetTempPath() : inputOptions.TempFolder, "TwitchDownloader", "chatupdatecache"); + + // Clear working directory if it already exists + if (Directory.Exists(cacheFolder)) + Directory.Delete(cacheFolder, true); + + // Thirdparty emotes + if (chatRoot.embeddedData.thirdParty == null || inputOptions.UpdateOldEmbeds) + { + chatRoot.embeddedData.thirdParty = new List(); + } + Console.WriteLine("Input third party emote count: " + chatRoot.embeddedData.thirdParty.Count); + List thirdPartyEmotes = new List(); + thirdPartyEmotes = Task.Run(() => TwitchHelper.GetThirdPartyEmotes(chatRoot.streamer.id, cacheFolder, bttv: inputOptions.BttvEmotes, ffz: inputOptions.FfzEmotes, stv: inputOptions.StvEmotes, embeddedData: chatRoot.embeddedData)).Result; + foreach (TwitchEmote emote in thirdPartyEmotes) + { + EmbedEmoteData newEmote = new EmbedEmoteData(); + newEmote.id = emote.Id; + newEmote.imageScale = emote.ImageScale; + newEmote.data = emote.ImageData; + newEmote.name = emote.Name; + newEmote.width = emote.Width / emote.ImageScale; + newEmote.height = emote.Height / emote.ImageScale; + chatRoot.embeddedData.thirdParty.Add(newEmote); + } + Console.WriteLine("Output third party emote count: " + chatRoot.embeddedData.thirdParty.Count); + + // Firstparty emotes + if (chatRoot.embeddedData.firstParty == null || inputOptions.UpdateOldEmbeds) + { + chatRoot.embeddedData.firstParty = new List(); + } + Console.WriteLine("Input first party emote count: " + chatRoot.embeddedData.firstParty.Count); + List firstPartyEmotes = new List(); + firstPartyEmotes = Task.Run(() => TwitchHelper.GetEmotes(chatRoot.comments, cacheFolder, embeddedData: chatRoot.embeddedData)).Result; + foreach (TwitchEmote emote in firstPartyEmotes) + { + EmbedEmoteData newEmote = new EmbedEmoteData(); + newEmote.id = emote.Id; + newEmote.imageScale = emote.ImageScale; + newEmote.data = emote.ImageData; + newEmote.width = emote.Width / emote.ImageScale; + newEmote.height = emote.Height / emote.ImageScale; + chatRoot.embeddedData.firstParty.Add(newEmote); + } + Console.WriteLine("Output third party emote count: " + chatRoot.embeddedData.firstParty.Count); + + // Twitch badges + if (chatRoot.embeddedData.twitchBadges == null || inputOptions.UpdateOldEmbeds) + { + chatRoot.embeddedData.twitchBadges = new List(); + } + Console.WriteLine("Input twitch badge count: " + chatRoot.embeddedData.twitchBadges.Count); + List twitchBadges = new List(); + twitchBadges = Task.Run(() => TwitchHelper.GetChatBadges(chatRoot.streamer.id, cacheFolder, embeddedData: chatRoot.embeddedData)).Result; + foreach (ChatBadge badge in twitchBadges) + { + EmbedChatBadge newBadge = new EmbedChatBadge(); + newBadge.name = badge.Name; + newBadge.versions = badge.VersionsData; + chatRoot.embeddedData.twitchBadges.Add(newBadge); + } + Console.WriteLine("Output twitch badge count: " + chatRoot.embeddedData.twitchBadges.Count); + + // Twitch bits / cheers + if (chatRoot.embeddedData.twitchBits == null || inputOptions.UpdateOldEmbeds) + { + chatRoot.embeddedData.twitchBits = new List(); + } + Console.WriteLine("Input twitch bit count: " + chatRoot.embeddedData.twitchBits.Count); + List twitchBits = new List(); + twitchBits = Task.Run(() => TwitchHelper.GetBits(cacheFolder, chatRoot.streamer.id.ToString(), embeddedData: chatRoot.embeddedData)).Result; + foreach (CheerEmote bit in twitchBits) + { + EmbedCheerEmote newBit = new EmbedCheerEmote(); + newBit.prefix = bit.prefix; + newBit.tierList = new Dictionary(); + foreach (KeyValuePair emotePair in bit.tierList) + { + EmbedEmoteData newEmote = new EmbedEmoteData(); + newEmote.id = emotePair.Value.Id; + newEmote.imageScale = emotePair.Value.ImageScale; + newEmote.data = emotePair.Value.ImageData; + newEmote.name = emotePair.Value.Name; + newEmote.width = emotePair.Value.Width / emotePair.Value.ImageScale; + newEmote.height = emotePair.Value.Height / emotePair.Value.ImageScale; + newBit.tierList.Add(emotePair.Key, newEmote); + } + chatRoot.embeddedData.twitchBits.Add(newBit); + } + Console.WriteLine("Input twitch bit count: " + chatRoot.embeddedData.twitchBits.Count); + + // Finally save the output to file! + // TODO: maybe in the future we could also export as HTML here too? + if (outFormat == DownloadFormat.Json) + { + using (TextWriter writer = File.CreateText(inputOptions.OutputFile)) + { + var serializer = new Newtonsoft.Json.JsonSerializer(); + serializer.Serialize(writer, chatRoot); + } + } + + // Clear our working directory, it's highly unlikely we would reuse it anyways + if (Directory.Exists(cacheFolder)) + Directory.Delete(cacheFolder, true); + } + } +} diff --git a/TwitchDownloaderCLI/Modes/RenderChat.cs b/TwitchDownloaderCLI/Modes/RenderChat.cs index c617819b..2d361d67 100644 --- a/TwitchDownloaderCLI/Modes/RenderChat.cs +++ b/TwitchDownloaderCLI/Modes/RenderChat.cs @@ -58,7 +58,8 @@ internal static void Render(ChatRenderArgs inputOptions) TempFolder = inputOptions.TempFolder, SubMessages = (bool)inputOptions.SubMessages, ChatBadges = (bool)inputOptions.ChatBadges, - Timestamp = inputOptions.Timestamp + Timestamp = inputOptions.Timestamp, + Offline = (bool)inputOptions.Offline, }; if (renderOptions.GenerateMask && renderOptions.BackgroundColor.Alpha == 255) diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs index ba023896..07727d7b 100644 --- a/TwitchDownloaderCLI/Program.cs +++ b/TwitchDownloaderCLI/Program.cs @@ -32,22 +32,21 @@ static void Main(string[] args) } string[] preParsedArgs; - if (args.Any(x => x.Equals("-m") || x.Equals("--mode"))) + if (args.Any(x => x is "-m" or "--mode" or "--embed-emotes")) { - // Old -m/--mode syntax was used, print an info message and convert to verb syntax - Console.WriteLine("[INFO] The program has switched from --mode to verbs (like \"git \"), consider using verbs instead." + - " Run \"{0} help\" for more information.", processFileName); - preParsedArgs = PreParseArgs.Process(PreParseArgs.ConvertFromOldSyntax(args)); + // A legacy syntax was used, convert to new syntax + preParsedArgs = PreParseArgs.Process(PreParseArgs.ConvertFromOldSyntax(args, processFileName)); } else { preParsedArgs = PreParseArgs.Process(args); } - Parser.Default.ParseArguments(preParsedArgs) + Parser.Default.ParseArguments(preParsedArgs) .WithParsed(DownloadVideo.Download) .WithParsed(DownloadClip.Download) .WithParsed(DownloadChat.Download) + .WithParsed(DownloadChatUpdater.Update) .WithParsed(RenderChat.Render) .WithParsed(FfmpegHandler.ParseArgs) .WithParsed(CacheHandler.ParseArgs) diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index 9a44171f..9dc00650 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -5,6 +5,7 @@ A cross platform command line tool that can do the main functions of the GUI pro - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) - [Arguments for mode chatrender](#arguments-for-mode-chatrender) + - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate) - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) - [Arguments for mode cache](#arguments-for-mode-cache) - [Example commands](#example-commands) @@ -73,17 +74,17 @@ Time in seconds to crop beginning. For example if I had a 10 second stream but o **-e/-\-ending** Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. -**-E/-\-embed-emotes** -(Default: false) Embeds emotes into the JSON file so in the future when an emote is removed from Twitch or a 3rd party, it will still render correctly. Useful for archival purposes, file size will be larger. +**-E/-\-embed-images** +(Default: false) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger. **-\-bttv** -(Default: true) BTTV emote embedding. Requires `-E / --embed-emotes`. +(Default: true) BTTV emote embedding. Requires `-E / --embed-images`. **-\-ffz** -(Default: true) FFZ emote embedding. Requires `-E / --embed-emotes`. +(Default: true) FFZ emote embedding. Requires `-E / --embed-images`. **-\-stv** -(Default: true) 7TV emote embedding. Requires `-E / --embed-emotes`. +(Default: true) 7TV emote embedding. Requires `-E / --embed-images`. **-\-timestamp** (Default: false) Enable timestamps @@ -162,7 +163,7 @@ File the program will output to. (Default: 0.2) Time in seconds to update chat render output. **-\-input-args** - (Default: -framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -) Input (pass1) arguments for ffmpeg chat render. +(Default: -framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -) Input (pass1) arguments for ffmpeg chat render. **-\-output-args** (Default: -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}") Output (pass2) arguments for ffmpeg chat render. @@ -182,6 +183,9 @@ Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128` +**-\-offline** +Render completely offline, using only resources embedded emotes, badges, and bits in the input json. + **-\-ffmpeg-path** Path to ffmpeg executable. @@ -189,6 +193,33 @@ Path to ffmpeg executable. Path to temporary folder for cache. +## Arguments for mode chatupdate + +**-i/-\-input (REQUIRED)** +Path to input file. Valid extensions are json + +**-o/-\-output (REQUIRED)** +Path to output file. Extension should match the input. + +**-E/-\-embed-missing** +(Default: true) Embed missing emotes, badges, and bits. Already embedded images will be untouched. + +**-U/-\-update-old** +(Default: false) Update old emotes, badges, and bits to the current. All embedded images will be overwritten! + +**-\-bttv** +(Default: true) Enable embedding BTTV emotes. + +**-\-ffz** +(Default: true) Enable embedding FFZ emotes. + +**-\-stv** +(Default: true) Enable embedding 7TV emotes. + +**-\-temp-path** +Path to temporary folder for cache. + + ## Arguments for mode ffmpeg Manage standalone ffmpeg @@ -219,7 +250,7 @@ Download a Chat (plain text with timestamps) TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt Download a Chat (JSON with embeded emotes from Twitch and Bttv) - TwitchDownloaderCLI chatdownload --id 612942303 --embed-emotes --bttv=true --ffz=false --stv=false -o chat.json + TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json Render a chat with defaults TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4 diff --git a/TwitchDownloaderCLI/Tools/PreParseArgs.cs b/TwitchDownloaderCLI/Tools/PreParseArgs.cs index b23b1fbb..4ecc3b5d 100644 --- a/TwitchDownloaderCLI/Tools/PreParseArgs.cs +++ b/TwitchDownloaderCLI/Tools/PreParseArgs.cs @@ -1,4 +1,8 @@ -namespace TwitchDownloaderCLI.Tools +using System; +using System.Collections.Generic; +using System.Linq; + +namespace TwitchDownloaderCLI.Tools { internal static class PreParseArgs { @@ -10,30 +14,50 @@ internal static string[] Process(string[] args) } /// - /// Converts an argument array that uses the old -m/--mode syntax to verb syntax + /// Converts an argument array that uses any legacy syntax to the current syntax /// /// - /// The same array but using a verb instead of -m - internal static string[] ConvertFromOldSyntax(string[] args) + /// The same array but using current syntax instead + internal static string[] ConvertFromOldSyntax(string[] args, string processFileName) { int argsLength = args.Length; - string[] processedArgs = new string[argsLength - 1]; + List processedArgs = args.ToList(); - int j = 1; - for (int i = 0; i < argsLength; i++) + if (args.Any(x => x.Equals("--embed-emotes"))) { - if (args[i].Equals("-m") || args[i].Equals("--mode")) + Console.WriteLine("[INFO] The program has switched from --embed-emotes to --embed-images OR -E, consider using those instead. Run \'{0} help\' for more information.", processFileName); + for (int i = 0; i < argsLength; i++) { - // Copy the runmode to the verb position - processedArgs[0] = args[i + 1]; - i++; - continue; + if (processedArgs[i].Equals("--embed-emotes")) + { + processedArgs[i] = "-E"; + break; + } } - processedArgs[j] = args[i]; - j++; } - return processedArgs; + // This must always be performed last + if (args.Any(x => x.Equals("-m") || x.Equals("--mode"))) + { + Console.WriteLine("[INFO] The program has switched from --mode to verbs (like \'git \'), consider using verbs instead. Run \'{0} help\' for more information.", processFileName); + int j = 1; + for (int i = 0; i < argsLength; i++) + { + if (processedArgs[i].Equals("-m") || processedArgs[i].Equals("--mode")) + { + // Copy the runmode to the verb position + processedArgs[0] = processedArgs[i + 1]; + i++; + continue; + } + processedArgs[j] = processedArgs[i]; + j++; + } + // Remove last element as it will be a duplicate of second last element + processedArgs.RemoveAt(processedArgs.Count - 1); + } + + return processedArgs.ToArray(); } } } diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index f8397d33..c941cb19 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -1,4 +1,7 @@ -using Newtonsoft.Json; +using NeoSmart.Unicode; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SkiaSharp; using System; using System.Collections.Generic; using System.IO; @@ -324,19 +327,25 @@ public async Task DownloadAsync(IProgress progress, Cancellation comments = commentsSet.DistinctBy(x => x._id).ToList(); chatRoot.comments = comments; - if (downloadOptions.EmbedEmotes && (downloadOptions.DownloadFormat == DownloadFormat.Json || downloadOptions.DownloadFormat == DownloadFormat.Html)) + if (downloadOptions.EmbedData && (downloadOptions.DownloadFormat == DownloadFormat.Json || downloadOptions.DownloadFormat == DownloadFormat.Html)) { - progress.Report(new ProgressReport() { reportType = ReportType.Message, data = "Downloading + Embedding Emotes" }); - chatRoot.emotes = new Emotes(); - List firstParty = new List(); - List thirdParty = new List(); + progress.Report(new ProgressReport() { reportType = ReportType.Message, data = "Downloading + Embedding Images" }); + chatRoot.embeddedData = new EmbeddedData(); + List firstPartyReturnList = new List(); + List thirdPartyReturnList = new List(); + List badgesReturnList = new List(); + List bitsReturnList = new List(); string cacheFolder = Path.Combine(Path.GetTempPath(), "TwitchDownloader", "cache"); List thirdPartyEmotes = new List(); List firstPartyEmotes = new List(); + List twitchBadges = new List(); + List twitchBits = new List(); thirdPartyEmotes = await TwitchHelper.GetThirdPartyEmotes(chatRoot.streamer.id, cacheFolder, bttv: downloadOptions.BttvEmotes, ffz: downloadOptions.FfzEmotes, stv: downloadOptions.StvEmotes); firstPartyEmotes = await TwitchHelper.GetEmotes(comments, cacheFolder); + twitchBadges = await TwitchHelper.GetChatBadges(chatRoot.streamer.id, cacheFolder); + twitchBits = await TwitchHelper.GetBits(cacheFolder, chatRoot.streamer.id.ToString()); foreach (TwitchEmote emote in thirdPartyEmotes) { @@ -347,7 +356,7 @@ public async Task DownloadAsync(IProgress progress, Cancellation newEmote.name = emote.Name; newEmote.width = emote.Width / emote.ImageScale; newEmote.height = emote.Height / emote.ImageScale; - thirdParty.Add(newEmote); + thirdPartyReturnList.Add(newEmote); } foreach (TwitchEmote emote in firstPartyEmotes) { @@ -357,11 +366,38 @@ public async Task DownloadAsync(IProgress progress, Cancellation newEmote.data = emote.ImageData; newEmote.width = emote.Width / emote.ImageScale; newEmote.height = emote.Height / emote.ImageScale; - firstParty.Add(newEmote); + firstPartyReturnList.Add(newEmote); + } + foreach (ChatBadge badge in twitchBadges) + { + EmbedChatBadge newBadge = new EmbedChatBadge(); + newBadge.name = badge.Name; + newBadge.versions = badge.VersionsData; + badgesReturnList.Add(newBadge); + } + foreach (CheerEmote bit in twitchBits) + { + EmbedCheerEmote newBit = new EmbedCheerEmote(); + newBit.prefix = bit.prefix; + newBit.tierList = new Dictionary(); + foreach (KeyValuePair emotePair in bit.tierList) + { + EmbedEmoteData newEmote = new EmbedEmoteData(); + newEmote.id = emotePair.Value.Id; + newEmote.imageScale = emotePair.Value.ImageScale; + newEmote.data = emotePair.Value.ImageData; + newEmote.name = emotePair.Value.Name; + newEmote.width = emotePair.Value.Width / emotePair.Value.ImageScale; + newEmote.height = emotePair.Value.Height / emotePair.Value.ImageScale; + newBit.tierList.Add(emotePair.Key, newEmote); + } + bitsReturnList.Add(newBit); } - chatRoot.emotes.thirdParty = thirdParty; - chatRoot.emotes.firstParty = firstParty; + chatRoot.embeddedData.thirdParty = thirdPartyReturnList; + chatRoot.embeddedData.firstParty = firstPartyReturnList; + chatRoot.embeddedData.twitchBadges = badgesReturnList; + chatRoot.embeddedData.twitchBits = bitsReturnList; } if (downloadOptions.DownloadFormat == DownloadFormat.Json) @@ -415,9 +451,9 @@ public async Task DownloadAsync(IProgress progress, Cancellation { if (!thirdEmoteData.ContainsKey(item.Code)) { - if (downloadOptions.EmbedEmotes) + if (downloadOptions.EmbedData) { - EmbedEmoteData embedEmoteData = chatRoot.emotes.thirdParty.FirstOrDefault(x => x.id == item.Id); + EmbedEmoteData embedEmoteData = chatRoot.embeddedData.thirdParty.FirstOrDefault(x => x.id == item.Id); if (embedEmoteData != null) { embedEmoteData.url = item.ImageUrl.Replace("[scale]", "1"); @@ -444,13 +480,13 @@ public async Task DownloadAsync(IProgress progress, Cancellation finalString.AppendLine(HttpUtility.HtmlEncode(Path.GetFileNameWithoutExtension(downloadOptions.Filename))); break; case "/* [CUSTOM CSS] */": - if (downloadOptions.EmbedEmotes) + if (downloadOptions.EmbedData) { - foreach (var emote in chatRoot.emotes.firstParty) + foreach (var emote in chatRoot.embeddedData.firstParty) { finalString.AppendLine(".first-" + emote.id + " { content:url(\"data:image/png;base64, " + Convert.ToBase64String(emote.data) + "\"); }"); } - foreach (var emote in chatRoot.emotes.thirdParty) + foreach (var emote in chatRoot.embeddedData.thirdParty) { finalString.AppendLine(".third-" + emote.id + " { content:url(\"data:image/png;base64, " + Convert.ToBase64String(emote.data) + "\"); }"); } @@ -461,7 +497,7 @@ public async Task DownloadAsync(IProgress progress, Cancellation { TimeSpan time = new TimeSpan(0, 0, (int)comment.content_offset_seconds); string timestamp = time.ToString(@"h\:mm\:ss"); - finalString.Append($"
[{timestamp}] {(comment.commenter.display_name.Any(x => x > 127) ? ($"{comment.commenter.display_name} ({comment.commenter.name})") : comment.commenter.display_name)}: {GetMessageHtml(downloadOptions.EmbedEmotes, thirdEmoteData, chatRoot, comment)}
\n"); + finalString.Append($"
[{timestamp}] {(comment.commenter.display_name.Any(x => x > 127) ? ($"{comment.commenter.display_name} ({comment.commenter.name})") : comment.commenter.display_name)}: {GetMessageHtml(downloadOptions.EmbedData, thirdEmoteData, chatRoot, comment)}
\n"); } break; default: @@ -533,7 +569,7 @@ private string GetMessageHtml(bool embedEmotes, Dictionary x.id == fragment.emoticon.emoticon_id)) + if (embedEmotes && chatRoot.embeddedData.firstParty.Any(x => x.id == fragment.emoticon.emoticon_id)) { message.Append($"
{fragment.text}
"); } diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index 148160a3..f7070cb9 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -44,10 +44,10 @@ public ChatRenderer(ChatRenderOptions chatRenderOptions) public async Task RenderVideoAsync(IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport() { reportType = ReportType.Message, data = "Fetching Images" }); - Task> badgeTask = Task.Run(() => TwitchHelper.GetChatBadges(chatRoot.streamer.id, renderOptions.TempFolder)); - Task> emoteTask = Task.Run(() => TwitchHelper.GetEmotes(chatRoot.comments, renderOptions.TempFolder, chatRoot.emotes)); - Task> emoteThirdTask = Task.Run(() => TwitchHelper.GetThirdPartyEmotes(chatRoot.streamer.id, renderOptions.TempFolder, chatRoot.emotes, renderOptions.BttvEmotes, renderOptions.FfzEmotes, renderOptions.StvEmotes)); - Task> cheerTask = Task.Run(() => TwitchHelper.GetBits(renderOptions.TempFolder, chatRoot.streamer.id.ToString())); + Task> badgeTask = Task.Run(() => TwitchHelper.GetChatBadges(chatRoot.streamer.id, renderOptions.TempFolder, chatRoot.embeddedData, offline: renderOptions.Offline)); + Task> emoteTask = Task.Run(() => TwitchHelper.GetEmotes(chatRoot.comments, renderOptions.TempFolder, chatRoot.embeddedData, offline: renderOptions.Offline)); + Task> emoteThirdTask = Task.Run(() => TwitchHelper.GetThirdPartyEmotes(chatRoot.streamer.id, renderOptions.TempFolder, chatRoot.embeddedData, renderOptions.BttvEmotes, renderOptions.FfzEmotes, renderOptions.StvEmotes, offline: renderOptions.Offline)); + Task> cheerTask = Task.Run(() => TwitchHelper.GetBits(renderOptions.TempFolder, chatRoot.streamer.id.ToString(), chatRoot.embeddedData, offline: renderOptions.Offline)); Task> emojiTask = Task.Run(() => TwitchHelper.GetTwitterEmojis(renderOptions.TempFolder)); await Task.WhenAll(badgeTask, emoteTask, emoteThirdTask, cheerTask, emojiTask); @@ -1074,10 +1074,23 @@ private static bool IsRightToLeft(string message) } public async Task ParseJson() { - using FileStream fs = new FileStream(renderOptions.InputFile, FileMode.Open, FileAccess.Read); - using var jsonDocument = JsonDocument.Parse(fs); + chatRoot = Task.Run(() => ParseJsonStatic(renderOptions.InputFile)).Result; - if (jsonDocument.RootElement.TryGetProperty("streamer", out JsonElement streamerJson)) + if (chatRoot.streamer == null) + { + chatRoot.streamer = new Streamer(); + chatRoot.streamer.id = int.Parse(chatRoot.comments.First().channel_id); + chatRoot.streamer.name = await TwitchHelper.GetStreamerName(chatRoot.streamer.id); + } + + return chatRoot; + } + + public static async Task ParseJsonStatic(string inputJson) + { + ChatRoot chatRoot = new ChatRoot(); + + using (FileStream fs = new FileStream(inputJson, FileMode.Open, FileAccess.Read)) { chatRoot.streamer = streamerJson.Deserialize(); } @@ -1085,7 +1098,32 @@ public async Task ParseJson() { if (videoJson.TryGetProperty("start", out _) && videoJson.TryGetProperty("end", out _)) { - chatRoot.video = videoJson.Deserialize(); + if (jsonDocument.RootElement.TryGetProperty("streamer", out JsonElement streamerJson)) + { + chatRoot.streamer = streamerJson.Deserialize(); + } + + if (jsonDocument.RootElement.TryGetProperty("video", out JsonElement videoJson)) + { + if (videoJson.TryGetProperty("start", out JsonElement videoStartJson) && videoJson.TryGetProperty("end", out JsonElement videoEndJson)) + { + chatRoot.video = videoJson.Deserialize(); + } + } + + if (jsonDocument.RootElement.TryGetProperty("embeddedData", out JsonElement embedDataJson)) + { + chatRoot.embeddedData = embedDataJson.Deserialize(); + } + else if (jsonDocument.RootElement.TryGetProperty("emotes", out JsonElement emotesJson)) + { + chatRoot.embeddedData = emotesJson.Deserialize(); + } + + if (jsonDocument.RootElement.TryGetProperty("comments", out JsonElement commentsJson)) + { + chatRoot.comments = commentsJson.Deserialize>(); + } } } if (jsonDocument.RootElement.TryGetProperty("emotes", out JsonElement emotesJson)) @@ -1097,13 +1135,6 @@ public async Task ParseJson() chatRoot.comments = commentsJson.Deserialize>(); } - if (chatRoot.streamer == null) - { - chatRoot.streamer = new Streamer(); - chatRoot.streamer.id = int.Parse(chatRoot.comments.First().channel_id); - chatRoot.streamer.name = await TwitchHelper.GetStreamerName(chatRoot.streamer.id); - } - return chatRoot; } } diff --git a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs index 6971cea6..15038ef3 100644 --- a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs +++ b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs @@ -16,7 +16,7 @@ public class ChatDownloadOptions public bool CropEnding { get; set; } public double CropEndingTime { get; set; } public bool Timestamp { get; set; } - public bool EmbedEmotes { get; set; } + public bool EmbedData { get; set; } public bool BttvEmotes { get; set; } public bool FfzEmotes { get; set; } public bool StvEmotes { get; set; } diff --git a/TwitchDownloaderCore/Options/ChatRenderOptions.cs b/TwitchDownloaderCore/Options/ChatRenderOptions.cs index a1b551b4..f64363fa 100644 --- a/TwitchDownloaderCore/Options/ChatRenderOptions.cs +++ b/TwitchDownloaderCore/Options/ChatRenderOptions.cs @@ -118,5 +118,6 @@ public int AscentIndentWidth return (int)(24 * ReferenceScale); } } + public bool Offline { get; set; } } } diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs index 10dcf69f..433cc4e2 100644 --- a/TwitchDownloaderCore/TwitchHelper.cs +++ b/TwitchDownloaderCore/TwitchHelper.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using NeoSmart.Unicode; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SkiaSharp; using System; @@ -6,11 +7,13 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Net; using System.Net.Http; using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Xml.Linq; using TwitchDownloaderCore.Properties; using TwitchDownloaderCore.TwitchObjects; @@ -130,7 +133,7 @@ public static async Task GetThirdPartyEmoteData(string streamerId BTTV.Merge(bttvChannel["channelEmotes"]); BTTV.Merge(bttvChannel["sharedEmotes"]); } - catch { } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } foreach (var emote in BTTV) @@ -154,7 +157,7 @@ public static async Task GetThirdPartyEmoteData(string streamerId { FFZ.Merge(JArray.Parse(await httpClient.GetStringAsync("https://api.betterttv.net/3/cached/frankerfacez/users/twitch/" + streamerId))); } - catch { } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } foreach (var emote in FFZ) @@ -180,7 +183,7 @@ public static async Task GetThirdPartyEmoteData(string streamerId JObject streamerEmoteObject = JObject.Parse(await httpClient.GetStringAsync(string.Format("https://7tv.io/v3/users/twitch/{0}", streamerId))); stvEmotes.Merge((JArray)streamerEmoteObject["emote_set"]["emotes"]); } - catch { } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } foreach (var stvEmote in stvEmotes) @@ -226,20 +229,15 @@ public static async Task GetThirdPartyEmoteData(string streamerId return emoteReponse; } - public static async Task> GetThirdPartyEmotes(int streamerId, string cacheFolder, Emotes embededEmotes = null, bool bttv = true, bool ffz = true, bool stv = true) + public static async Task> GetThirdPartyEmotes(int streamerId, string cacheFolder, EmbeddedData embeddedData = null, bool bttv = true, bool ffz = true, bool stv = true, bool offline = false) { List returnList = new List(); List alreadyAdded = new List(); - string bttvFolder = Path.Combine(cacheFolder, "bttv"); - string ffzFolder = Path.Combine(cacheFolder, "ffz"); - string stvFolder = Path.Combine(cacheFolder, "stv"); - - EmoteResponse emoteDataResponse = await GetThirdPartyEmoteData(streamerId.ToString(), bttv, ffz, stv); - - if (embededEmotes != null) + // Load our embedded data from file + if (embeddedData != null && embeddedData.thirdParty != null) { - foreach (EmbedEmoteData emoteData in embededEmotes.thirdParty) + foreach (EmbedEmoteData emoteData in embeddedData.thirdParty) { try { @@ -247,10 +245,22 @@ public static async Task> GetThirdPartyEmotes(int streamerId, returnList.Add(newEmote); alreadyAdded.Add(emoteData.name); } - catch { } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } } + // Directly return if we are in offline, no need for a network request + if (offline) + { + return returnList; + } + + string bttvFolder = Path.Combine(cacheFolder, "bttv"); + string ffzFolder = Path.Combine(cacheFolder, "ffz"); + string stvFolder = Path.Combine(cacheFolder, "stv"); + + EmoteResponse emoteDataResponse = await GetThirdPartyEmoteData(streamerId.ToString(), bttv, ffz, stv); + if (bttv) { if (!Directory.Exists(bttvFolder)) @@ -260,11 +270,15 @@ public static async Task> GetThirdPartyEmotes(int streamerId, { if (alreadyAdded.Contains(emote.Code)) continue; - TwitchEmote newEmote = new TwitchEmote(await GetImage(bttvFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); - if (emote.IsZeroWidth) - newEmote.IsZeroWidth = true; - returnList.Add(newEmote); - alreadyAdded.Add(emote.Code); + try + { + TwitchEmote newEmote = new TwitchEmote(await GetImage(bttvFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); + if (emote.IsZeroWidth) + newEmote.IsZeroWidth = true; + returnList.Add(newEmote); + alreadyAdded.Add(emote.Code); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } } @@ -277,9 +291,13 @@ public static async Task> GetThirdPartyEmotes(int streamerId, { if (alreadyAdded.Contains(emote.Code)) continue; - TwitchEmote newEmote = new TwitchEmote(await GetImage(ffzFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); - returnList.Add(newEmote); - alreadyAdded.Add(emote.Code); + try + { + TwitchEmote newEmote = new TwitchEmote(await GetImage(ffzFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); + returnList.Add(newEmote); + alreadyAdded.Add(emote.Code); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } } @@ -292,18 +310,22 @@ public static async Task> GetThirdPartyEmotes(int streamerId, { if (alreadyAdded.Contains(emote.Code)) continue; - TwitchEmote newEmote = new TwitchEmote(await GetImage(stvFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); - if (emote.IsZeroWidth) - newEmote.IsZeroWidth = true; - returnList.Add(newEmote); - alreadyAdded.Add(emote.Code); + try + { + TwitchEmote newEmote = new TwitchEmote(await GetImage(stvFolder, emote.ImageUrl.Replace("[scale]", "2"), emote.Id, "2", emote.ImageType), EmoteProvider.ThirdParty, 2, emote.Id, emote.Code); + if (emote.IsZeroWidth) + newEmote.IsZeroWidth = true; + returnList.Add(newEmote); + alreadyAdded.Add(emote.Code); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } } return returnList; } - public static async Task> GetEmotes(List comments, string cacheFolder, Emotes embededEmotes = null) + public static async Task> GetEmotes(List comments, string cacheFolder, EmbeddedData embeddedData = null, bool offline = false) { List returnList = new List(); List alreadyAdded = new List(); @@ -313,9 +335,10 @@ public static async Task> GetEmotes(List comments, st if (!Directory.Exists(emoteFolder)) TwitchHelper.CreateDirectory(emoteFolder); - if (embededEmotes != null) + // Load our embedded emotes + if (embeddedData != null && embeddedData.firstParty != null) { - foreach (EmbedEmoteData emoteData in embededEmotes.firstParty) + foreach (EmbedEmoteData emoteData in embeddedData.firstParty) { try { @@ -327,6 +350,12 @@ public static async Task> GetEmotes(List comments, st } } + // Directly return if we are in offline, no need for a network request + if (offline) + { + return returnList; + } + foreach (var comment in comments) { if (comment.message.fragments == null) @@ -346,7 +375,7 @@ public static async Task> GetEmotes(List comments, st alreadyAdded.Add(id); returnList.Add(newEmote); } - catch (HttpRequestException ex) when (ex.Message.Contains("404")) + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { failedEmotes.Add(id); } @@ -358,10 +387,30 @@ public static async Task> GetEmotes(List comments, st return returnList; } - public static async Task> GetChatBadges(int streamerId, string cacheFolder) + public static async Task> GetChatBadges(int streamerId, string cacheFolder, EmbeddedData embeddedData = null, bool offline = false) { List returnList = new List(); + List alreadyAdded = new List(); + // Load our embedded data from file + if (embeddedData != null && embeddedData.twitchBadges != null) + { + foreach (EmbedChatBadge data in embeddedData.twitchBadges) + { + ChatBadge newBadge = new ChatBadge(data.name, data.versions); + returnList.Add(newBadge); + alreadyAdded.Add(data.name); + } + } + + // Directly return if we are in offline, no need for a network request + if (offline) + { + return returnList; + } + + // TODO: this currently only does twitch badges, but we could also support FFZ, BTTV, 7TV, etc badges! + // TODO: would want to make this configurable as we do for emotes though... JObject globalBadges = JObject.Parse(await httpClient.GetStringAsync("https://badges.twitch.tv/v1/badges/global/display")); JObject subBadges = JObject.Parse(await httpClient.GetStringAsync($"https://badges.twitch.tv/v1/badges/channels/{streamerId}/display")); @@ -373,30 +422,27 @@ public static async Task> GetChatBadges(int streamerId, string c { JProperty jBadgeProperty = badge.ToObject(); string name = jBadgeProperty.Name; - Dictionary versions = new Dictionary(); + if (alreadyAdded.Contains(name)) + continue; - foreach (var version in badge.First["versions"]) + try { - JProperty jVersionProperty = version.ToObject(); - string versionString = jVersionProperty.Name; - string downloadUrl = version.First["image_url_2x"].ToString(); - - try + Dictionary versions = new Dictionary(); + foreach (var version in badge.First["versions"]) { + JProperty jVersionProperty = version.ToObject(); + string versionString = jVersionProperty.Name; + string downloadUrl = version.First["image_url_2x"].ToString(); + string[] id_parts = downloadUrl.Split('/'); string id = id_parts[id_parts.Length - 2]; byte[] bytes = await GetImage(badgeFolder, downloadUrl, id, "2", "png"); - using MemoryStream ms = new MemoryStream(bytes); - //For some reason, twitch has corrupted images sometimes :) for example - //https://static-cdn.jtvnw.net/badges/v1/a9811799-dce3-475f-8feb-3745ad12b7ea/1 - SKBitmap badgeImage = SKBitmap.Decode(ms); - versions.Add(versionString, badgeImage); + versions.Add(versionString, bytes); } - catch (HttpRequestException) - { } - } - returnList.Add(new ChatBadge(name, versions)); + returnList.Add(new ChatBadge(name, versions)); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } return returnList; @@ -454,9 +500,33 @@ public static async Task> GetTwitterEmojis(string c return returnCache; } - public static async Task> GetBits(string cacheFolder, string channel_id = "") + public static async Task> GetBits(string cacheFolder, string channel_id = "", EmbeddedData embeddedData = null, bool offline = false) { - List returnCheermotes = new List(); + List returnList = new List(); + List alreadyAdded = new List(); + + // Load our embedded data from file + if (embeddedData != null && embeddedData.twitchBits != null) + { + foreach (EmbedCheerEmote data in embeddedData.twitchBits) + { + List> tierList = new List>(); + CheerEmote newEmote = new CheerEmote() { prefix = data.prefix, tierList = tierList }; + foreach (KeyValuePair tier in data.tierList) + { + TwitchEmote tierEmote = new TwitchEmote(tier.Value.data, EmoteProvider.FirstParty, tier.Value.imageScale, tier.Value.id, tier.Value.name); + tierList.Add(new KeyValuePair(tier.Key, tierEmote)); + } + returnList.Add(newEmote); + alreadyAdded.Add(data.prefix); + } + } + + // Directly return if we are in offline, no need for a network request + if (offline) + { + return returnList; + } var request = new HttpRequestMessage() { @@ -496,21 +566,26 @@ public static async Task> GetBits(string cacheFolder, string ch foreach (CheerNode node in group.nodes) { string prefix = node.prefix; - List> tierList = new List>(); - CheerEmote newEmote = new CheerEmote() { prefix = prefix, tierList = tierList }; - foreach (Tier tier in node.tiers) + if (alreadyAdded.Contains(prefix)) + continue; + try { - int minBits = tier.bits; - string url = templateURL.Replace("PREFIX", node.prefix.ToLower()).Replace("BACKGROUND", "dark").Replace("ANIMATION", "animated").Replace("TIER", tier.bits.ToString()).Replace("SCALE.EXTENSION", "2.gif"); - TwitchEmote emote = new TwitchEmote(await GetImage(bitFolder, url, node.id + tier.bits, "2", "gif"), EmoteProvider.FirstParty, 2, prefix + minBits, prefix + minBits); - tierList.Add(new KeyValuePair(minBits, emote)); - } - returnCheermotes.Add(newEmote); + List> tierList = new List>(); + CheerEmote newEmote = new CheerEmote() { prefix = prefix, tierList = tierList }; + foreach (Tier tier in node.tiers) + { + int minBits = tier.bits; + string url = templateURL.Replace("PREFIX", node.prefix.ToLower()).Replace("BACKGROUND", "dark").Replace("ANIMATION", "animated").Replace("TIER", tier.bits.ToString()).Replace("SCALE.EXTENSION", "2.gif"); + TwitchEmote emote = new TwitchEmote(await GetImage(bitFolder, url, node.id + tier.bits, "2", "gif"), EmoteProvider.FirstParty, 2, prefix + minBits, prefix + minBits); + tierList.Add(new KeyValuePair(minBits, emote)); + } + returnList.Add(newEmote); + } catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } } } - return returnCheermotes; + return returnList; } public static DirectoryInfo CreateDirectory(string path) diff --git a/TwitchDownloaderCore/TwitchObjects/ChatBadge.cs b/TwitchDownloaderCore/TwitchObjects/ChatBadge.cs index b708c9b3..244641ab 100644 --- a/TwitchDownloaderCore/TwitchObjects/ChatBadge.cs +++ b/TwitchDownloaderCore/TwitchObjects/ChatBadge.cs @@ -1,7 +1,11 @@ -using SkiaSharp; +using NeoSmart.Unicode; +using Newtonsoft.Json.Linq; +using SkiaSharp; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Http; using System.Text; namespace TwitchDownloaderCore.TwitchObjects @@ -23,12 +27,23 @@ public class ChatBadge { public string Name; public Dictionary Versions; + public Dictionary VersionsData; public ChatBadgeType Type; - public ChatBadge(string name, Dictionary versions) + public ChatBadge(string name, Dictionary versions) { Name = name; - Versions = versions; + Versions = new Dictionary(); + VersionsData = versions; + + foreach (var version in versions) + { + using MemoryStream ms = new MemoryStream(version.Value); + //For some reason, twitch has corrupted images sometimes :) for example + //https://static-cdn.jtvnw.net/badges/v1/a9811799-dce3-475f-8feb-3745ad12b7ea/1 + SKBitmap badgeImage = SKBitmap.Decode(ms); + Versions.Add(version.Key, badgeImage); + } switch (name) { diff --git a/TwitchDownloaderCore/TwitchObjects/ChatRoot.cs b/TwitchDownloaderCore/TwitchObjects/ChatRoot.cs index f7c7e306..6ea3ed15 100644 --- a/TwitchDownloaderCore/TwitchObjects/ChatRoot.cs +++ b/TwitchDownloaderCore/TwitchObjects/ChatRoot.cs @@ -1,5 +1,4 @@ -using Newtonsoft.Json; -using System; +using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -97,10 +96,24 @@ public class EmbedEmoteData public int height { get; set; } } -public class Emotes +public class EmbedChatBadge +{ + public string name { get; set; } + public Dictionary versions { get; set; } +} + +public class EmbedCheerEmote +{ + public string prefix { get; set; } + public Dictionary tierList { get; set; } +} + +public class EmbeddedData { public List thirdParty { get; set; } public List firstParty { get; set; } + public List twitchBadges { get; set; } + public List twitchBits { get; set; } } public class CommentResponse @@ -118,5 +131,5 @@ public class ChatRoot [JsonPropertyOrder(2)] public List comments { get; set; } [JsonPropertyOrder(3)] - public Emotes emotes { get; set; } + public EmbeddedData embeddedData { get; set; } } \ No newline at end of file diff --git a/TwitchDownloaderWPF/PageChatDownload.xaml.cs b/TwitchDownloaderWPF/PageChatDownload.xaml.cs index 382ba30c..414f8b21 100644 --- a/TwitchDownloaderWPF/PageChatDownload.xaml.cs +++ b/TwitchDownloaderWPF/PageChatDownload.xaml.cs @@ -235,7 +235,7 @@ public ChatDownloadOptions GetOptions(string filename) options.DownloadFormat = DownloadFormat.Text; options.Timestamp = true; - options.EmbedEmotes = (bool)checkEmbed.IsChecked; + options.EmbedData = (bool)checkEmbed.IsChecked; options.BttvEmotes = (bool)checkBttvEmbed.IsChecked; options.FfzEmotes = (bool)checkFfzEmbed.IsChecked; options.StvEmotes = (bool)checkStvEmbed.IsChecked; diff --git a/TwitchDownloaderWPF/PageChatRender.xaml.cs b/TwitchDownloaderWPF/PageChatRender.xaml.cs index 35a80e4a..ddfe28cb 100644 --- a/TwitchDownloaderWPF/PageChatRender.xaml.cs +++ b/TwitchDownloaderWPF/PageChatRender.xaml.cs @@ -86,7 +86,8 @@ public ChatRenderOptions GetOptions(string filename) FfmpegPath = "ffmpeg", TempFolder = Settings.Default.TempPath, SubMessages = (bool)checkSub.IsChecked, - ChatBadges = (bool)checkBadge.IsChecked + ChatBadges = (bool)checkBadge.IsChecked, + Offline = false }; foreach (var item in comboBadges.SelectedItems) { diff --git a/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs b/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs index ae03bc1a..33926d8e 100644 --- a/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs +++ b/TwitchDownloaderWPF/WindowQueueOptions.xaml.cs @@ -111,7 +111,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e) chatOptions.DownloadFormat = DownloadFormat.Html; else chatOptions.DownloadFormat = DownloadFormat.Text; - chatOptions.EmbedEmotes = (bool)checkEmbed.IsChecked; + chatOptions.EmbedData = (bool)checkEmbed.IsChecked; chatOptions.Filename = Path.Combine(folderPath, MainWindow.GetFilename(Settings.Default.TemplateChat, downloadTask.Info.Title, chatOptions.Id, vodPage.currentVideoTime, vodPage.textStreamer.Text) + "." + chatOptions.DownloadFormat); if (downloadOptions.CropBeginning) @@ -200,7 +200,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e) else chatOptions.DownloadFormat = DownloadFormat.Text; chatOptions.TimeFormat = TimestampFormat.Relative; - chatOptions.EmbedEmotes = (bool)checkEmbed.IsChecked; + chatOptions.EmbedData = (bool)checkEmbed.IsChecked; chatOptions.Filename = Path.Combine(folderPath, MainWindow.GetFilename(Settings.Default.TemplateChat, downloadTask.Info.Title, chatOptions.Id, clipPage.currentVideoTime, clipPage.textStreamer.Text) + "." + chatOptions.FileExtension); chatTask.DownloadOptions = chatOptions; @@ -378,7 +378,7 @@ private void btnQueue_Click(object sender, RoutedEventArgs e) downloadOptions.DownloadFormat = DownloadFormat.Html; else downloadOptions.DownloadFormat = DownloadFormat.Text; - downloadOptions.EmbedEmotes = (bool)checkEmbed.IsChecked; + downloadOptions.EmbedData = (bool)checkEmbed.IsChecked; downloadOptions.TimeFormat = TimestampFormat.Relative; downloadOptions.Id = dataList[i].Id; downloadOptions.CropBeginning = false;