Chatrender patch again (#513)

* Fix banned word list not working correctly

* Accept unlisted 7tv emotes

* Update banned word regex

* Add option to CLI to disallow unlisted stv emotes, add cancellation tokens to more methods

* Fix inconsistent capitalization of 7TV

* Query fallback cache only once using CollectionsMarshal

* Fix renders not starting on tick 0 improperly rendering unseen comments, remove redundant expensive operations (List<T>.ToArray, duplicate LINQ methods), optimize reversing RTL outlines (~6x faster & ~1/6th allocations), fix block art calculation using untrimmed string, cleanup

* Don't copy the frame if there are no animated emotes to draw

* Fix corrupted frames when also generating mask after 0d1fba8

* Revert portion of 335a7d5

* Fix flipped values for unlisted emotes and offline

* Fix #529 - animated emotes breaking due to integer overflow

* Use a long incase someone renders at 1fps

* Remove restricted comments outside of render loop

* Only do banned word case/culture invariation in regex, clarify dispersion & make default false

* Add logic to skip checking for third party emotes and emojis when we know it's not possible

* Make banned word regex not static

* Add CLI option to skip drive waiting

* Update README

TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs

@@ -31,7 +31,7 @@ public class ChatDownloadArgs
         [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download. Requires -E / --embed-images!")]
         public bool? FfzEmotes { get; set; }
 
-        [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download. Requires -E / --embed-images!")]
+        [Option("stv", Default = true, HelpText = "Enable 7TV embedding in chat download. Requires -E / --embed-images!")]
         public bool? StvEmotes { get; set; }
 
         [Option("timestamp-format", Default = TimestampFormat.Relative, HelpText = "Sets the timestamp format for .txt chat logs. Valid values are: Utc, Relative, and None")]

TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs

@@ -36,9 +36,12 @@ public class ChatRenderArgs
         [Option("ffz", Default = true, HelpText = "Enable FFZ emotes.")]
         public bool? FfzEmotes { get; set; }
 
-        [Option("stv", Default = true, HelpText = "Enable 7tv emotes.")]
+        [Option("stv", Default = true, HelpText = "Enable 7TV emotes.")]
         public bool? StvEmotes { get; set; }
 
+        [Option("allow-unlisted-emotes", Default = true, HelpText = "Allow unlisted 7TV emotes in the render.")]
+        public bool? AllowUnlistedEmotes { get; set; }
+
         [Option("sub-messages", Default = true, HelpText = "Enable sub/re-sub messages.")]
         public bool? SubMessages { get; set; }
 
@@ -90,7 +93,7 @@ public class ChatRenderArgs
         [Option("badge-filter", Default = 0, HelpText = "Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, 6 = no broadcaster or moderator badges.\r\nKey: Other = 1, Broadcaster = 2, Moderator = 4, VIP = 8, Subscriber = 16, Predictions = 32, NoAudio/NoVideo = 64, PrimeGaming = 128")]
         public int BadgeFilterMask { get; set; }
 
-        [Option("dispersion", Default = true, HelpText = "In November 2022 a Twitch API change made chat messages download only in whole seconds. If there are multiple messages on a second, they will be intelligently distributed over the second to improve chat flow.")]
+        [Option("dispersion", Default = false, HelpText = "In November 2022 a Twitch API change made chat messages download only in whole seconds. If there are multiple messages on a second, they will be intelligently distributed over the second to improve chat flow. Requires an update rate less than 1.0 for effective results.")]
         public bool? DisperseCommentOffsets { get; set; }
 
         [Option("offline", Default = false, HelpText = "Render completely offline using only embedded emotes, badges, and bits from the input json.")]
@@ -105,6 +108,9 @@ public class ChatRenderArgs
         [Option("verbose-ffmpeg", Default = false, HelpText = "Prints every message from ffmpeg.")]
         public bool LogFfmpegOutput { get; set; }
 
+        [Option("skip-drive-waiting", Default = false, HelpText = "Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives.")]
+        public bool SkipDriveWaiting { get; set; }
+
         [Option("scale-emote", Default = 1.0, HelpText = "Number to scale emote images.")]
         public double ScaleEmote { get; set; }
 

TwitchDownloaderCLI/Modes/RenderChat.cs

@@ -1,7 +1,6 @@
 using SkiaSharp;
 using System;
 using System.IO;
-using System.Linq;
 using System.Threading;
 using TwitchDownloaderCLI.Modes.Arguments;
 using TwitchDownloaderCLI.Tools;
@@ -40,6 +39,7 @@ private static ChatRenderOptions GetRenderOptions(ChatRenderArgs inputOptions)
                 BttvEmotes = (bool)inputOptions.BttvEmotes,
                 FfzEmotes = (bool)inputOptions.FfzEmotes,
                 StvEmotes = (bool)inputOptions.StvEmotes,
+                AllowUnlistedEmotes = (bool)inputOptions.AllowUnlistedEmotes,
                 Outline = inputOptions.Outline,
                 OutlineSize = inputOptions.OutlineSize,
                 Font = inputOptions.Font,
@@ -71,6 +71,7 @@ private static ChatRenderOptions GetRenderOptions(ChatRenderArgs inputOptions)
                 Timestamp = inputOptions.Timestamp,
                 Offline = inputOptions.Offline,
                 LogFfmpegOutput = inputOptions.LogFfmpegOutput,
+                SkipDriveWaiting = inputOptions.SkipDriveWaiting,
                 EmoteScale = inputOptions.ScaleEmote,
                 BadgeScale = inputOptions.ScaleBadge,
                 EmojiScale = inputOptions.ScaleEmoji,
@@ -110,7 +111,7 @@ private static ChatRenderOptions GetRenderOptions(ChatRenderArgs inputOptions)
 
             if (inputOptions.BannedWordsString != "")
             {
-                renderOptions.BannedWordsArray = inputOptions.BannedWordsString.ToLower().Split(',',
+                renderOptions.BannedWordsArray = inputOptions.BannedWordsString.Split(',',
                     StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
             }
 

TwitchDownloaderCLI/README.md

@@ -173,6 +173,9 @@ File the program will output to.
 **--stv**
 (Default: true) Enable 7TV emotes.
 
+**--allow-unlisted-emotes**
+(Default: true) Allow unlisted 7TV emotes in the render.
+
 **--sub-messages**
 (Default: true) Enable sub / re-sub messages.
 
@@ -227,19 +230,22 @@ File the program will output to.
 Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128`
 
 **--dispersion**
-(Default: true) In November 2022 a Twitch API change made chat messages download only in whole seconds. If there are multiple messages on a second, they will be intelligently distributed over the second to improve chat flow.
+(Default: false) In November 2022 a Twitch API change made chat messages download only in whole seconds. If there are multiple messages on a second, they will be intelligently distributed over the second to improve chat flow. Requires an update rate less than 1.0 for effective results.
 
 **--offline**
-Render completely offline using only embedded emotes, badges, and bits from the input json.
+(Default: false) Render completely offline using only embedded emotes, badges, and bits from the input json.
 
 **--ffmpeg-path**
-Path to ffmpeg executable.
+(Default: ) Path to ffmpeg executable.
 
 **--temp-path**
-Path to temporary folder for cache.
+(Default: ) Path to temporary folder for cache.
 
 **--verbose-ffmpeg**
-Prints every message from ffmpeg.
+(Default: false) Prints every message from ffmpeg.
+
+**--skip-drive-waiting**
+(Default: false) Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives. Skipping can result in 1-5% render speed increases.
 
 **--scale-emote**
 (Default: 1.0) Number to scale emote images.
@@ -332,7 +338,7 @@ Render a chat with custom ffmpeg arguments
 
 ## Additional Notes
 
-String arguments, such as output file, that contain spaces should be wrapped in double quotes <kbd>"</kbd> .
+String arguments, such as output file, that contain spaces should be wrapped in either single quotes <kbd>'</kbd> or double quotes <kbd>"</kbd> .
 
 Default true boolean flags must be assigned: `--default-true-flag=false`. Default false boolean flags should still be raised normally: `--default-false-flag`
 

TwitchDownloaderCore/Chat/ChatHtml.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
 using TwitchDownloaderCore.TwitchObjects;
@@ -15,12 +16,14 @@ public static class ChatHtml
         /// <summary>
         /// Serializes a chat Html file.
         /// </summary>
-        public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, bool embedData = true)
+        public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, bool embedData = true, CancellationToken cancellationToken = new())
         {
             ArgumentNullException.ThrowIfNull(filePath, nameof(filePath));
 
             Dictionary<string, EmbedEmoteData> thirdEmoteData = new();
-            await BuildThirdPartyDictionary(chatRoot, embedData, thirdEmoteData);
+            await BuildThirdPartyDictionary(chatRoot, embedData, thirdEmoteData, cancellationToken);
+
+            cancellationToken.ThrowIfCancellationRequested();
 
             string[] templateStrings = Properties.Resources.template.Split('\n');
             using var fs = File.Create(filePath);
@@ -61,9 +64,9 @@ public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, bool
             }
         }
 
-        private static async Task BuildThirdPartyDictionary(ChatRoot chatRoot, bool embedData, Dictionary<string, EmbedEmoteData> thirdEmoteData)
+        private static async Task BuildThirdPartyDictionary(ChatRoot chatRoot, bool embedData, Dictionary<string, EmbedEmoteData> thirdEmoteData, CancellationToken cancellationToken)
         {
-            EmoteResponse emotes = await TwitchHelper.GetThirdPartyEmoteData(chatRoot.streamer.id.ToString(), true, true, true);
+            EmoteResponse emotes = await TwitchHelper.GetThirdPartyEmoteData(chatRoot.streamer.id.ToString(), true, true, true, true, cancellationToken);
             List<EmoteResponseItem> itemList = new();
             itemList.AddRange(emotes.BTTV);
             itemList.AddRange(emotes.FFZ);

TwitchDownloaderCore/ChatDownloader.cs

@@ -431,7 +431,7 @@ public async Task DownloadAsync(IProgress<ProgressReport> progress, Cancellation
                     await ChatJson.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.Compression, cancellationToken);
                     break;
                 case ChatFormat.Html:
-                    await ChatHtml.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.EmbedData);
+                    await ChatHtml.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.EmbedData, cancellationToken);
                     break;
                 case ChatFormat.Text:
                     await ChatText.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.TimeFormat);