From 612ae2e894d9beac5b92772f2247dbee82927797 Mon Sep 17 00:00:00 2001 From: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com> Date: Mon, 12 May 2025 19:52:47 +0300 Subject: [PATCH] Add a setting to control whether to respect advisory rate limits (#1342) --- .../Commands/Base/DiscordCommandBase.cs | 13 +++- .../Discord/DiscordClient.cs | 63 +++++++++++-------- .../Discord/RateLimitPreference.cs | 36 +++++++++++ .../RateLimitPreferenceToStringConverter.cs | 28 +++++++++ .../Services/SettingsService.cs | 5 ++ .../Components/DashboardViewModel.cs | 2 +- .../ViewModels/Dialogs/SettingsViewModel.cs | 10 +++ .../Views/Dialogs/SettingsView.axaml | 19 ++++++ 8 files changed, 148 insertions(+), 28 deletions(-) create mode 100644 DiscordChatExporter.Core/Discord/RateLimitPreference.cs create mode 100644 DiscordChatExporter.Gui/Converters/RateLimitPreferenceToStringConverter.cs diff --git a/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs index 760c0932..a042bb0a 100644 --- a/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs @@ -27,8 +27,19 @@ public abstract class DiscordCommandBase : ICommand )] public bool IsBotToken { get; init; } = false; + [CommandOption( + "respect-rate-limits", + Description = "Whether to respect advisory rate limits. " + + "If disabled, only hard rate limits (i.e. 429 responses) will be respected." + )] + public bool ShouldRespectRateLimits { get; init; } = true; + private DiscordClient? _discordClient; - protected DiscordClient Discord => _discordClient ??= new DiscordClient(Token); + protected DiscordClient Discord => + _discordClient ??= new DiscordClient( + Token, + ShouldRespectRateLimits ? RateLimitPreference.RespectAll : RateLimitPreference.IgnoreAll + ); public virtual ValueTask ExecuteAsync(IConsole console) { diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs index 67cf18ec..a607113c 100644 --- a/DiscordChatExporter.Core/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs @@ -18,7 +18,10 @@ using JsonExtensions.Reading; namespace DiscordChatExporter.Core.Discord; -public class DiscordClient(string token) +public class DiscordClient( + string token, + RateLimitPreference rateLimitPreference = RateLimitPreference.RespectAll +) { private readonly Uri _baseUri = new("https://discord.com/api/v10/", UriKind.Absolute); private TokenKind? _resolvedTokenKind; @@ -47,33 +50,41 @@ public class DiscordClient(string token) innerCancellationToken ); - // If this was the last request available before hitting the rate limit, - // wait out the reset time so that future requests can succeed. - // This may add an unnecessary delay in case the user doesn't intend to - // make any more requests, but implementing a smarter solution would - // require properly keeping track of Discord's global/per-route/per-resource - // rate limits and that's just way too much effort. - // https://discord.com/developers/docs/topics/rate-limits - var remainingRequestCount = response - .Headers.TryGetValue("X-RateLimit-Remaining") - ?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture)); - - var resetAfterDelay = response - .Headers.TryGetValue("X-RateLimit-Reset-After") - ?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture)) - .Pipe(TimeSpan.FromSeconds); - - if (remainingRequestCount <= 0 && resetAfterDelay is not null) + // Discord has advisory rate limits (communicated via response headers), but they are typically + // way stricter than the actual rate limits enforced by the server. + // The user may choose to ignore the advisory rate limits and only retry on hard rate limits, + // if they want to prioritize speed over compliance (and safety of their account/bot). + // https://github.com/Tyrrrz/DiscordChatExporter/issues/1021 + if (rateLimitPreference.IsRespectedFor(tokenKind)) { - var delay = - // Adding a small buffer to the reset time reduces the chance of getting - // rate limited again, because it allows for more requests to be released. - (resetAfterDelay.Value + TimeSpan.FromSeconds(1)) - // Sometimes Discord returns an absurdly high value for the reset time, which - // is not actually enforced by the server. So we cap it at a reasonable value. - .Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60)); + var remainingRequestCount = response + .Headers.TryGetValue("X-RateLimit-Remaining") + ?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture)); - await Task.Delay(delay, innerCancellationToken); + var resetAfterDelay = response + .Headers.TryGetValue("X-RateLimit-Reset-After") + ?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture)) + .Pipe(TimeSpan.FromSeconds); + + // If this was the last request available before hitting the rate limit, + // wait out the reset time so that future requests can succeed. + // This may add an unnecessary delay in case the user doesn't intend to + // make any more requests, but implementing a smarter solution would + // require properly keeping track of Discord's global/per-route/per-resource + // rate limits and that's just way too much effort. + // https://discord.com/developers/docs/topics/rate-limits + if (remainingRequestCount <= 0 && resetAfterDelay is not null) + { + var delay = + // Adding a small buffer to the reset time reduces the chance of getting + // rate limited again, because it allows for more requests to be released. + (resetAfterDelay.Value + TimeSpan.FromSeconds(1)) + // Sometimes Discord returns an absurdly high value for the reset time, which + // is not actually enforced by the server. So we cap it at a reasonable value. + .Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60)); + + await Task.Delay(delay, innerCancellationToken); + } } return response; diff --git a/DiscordChatExporter.Core/Discord/RateLimitPreference.cs b/DiscordChatExporter.Core/Discord/RateLimitPreference.cs new file mode 100644 index 00000000..62d9acbc --- /dev/null +++ b/DiscordChatExporter.Core/Discord/RateLimitPreference.cs @@ -0,0 +1,36 @@ +using System; + +namespace DiscordChatExporter.Core.Discord; + +[Flags] +public enum RateLimitPreference +{ + IgnoreAll = 0, + RespectForUserTokens = 0b1, + RespectForBotTokens = 0b10, + RespectAll = RespectForUserTokens | RespectForBotTokens, +} + +public static class RateLimitPreferenceExtensions +{ + internal static bool IsRespectedFor( + this RateLimitPreference rateLimitPreference, + TokenKind tokenKind + ) => + tokenKind switch + { + TokenKind.User => (rateLimitPreference & RateLimitPreference.RespectForUserTokens) != 0, + TokenKind.Bot => (rateLimitPreference & RateLimitPreference.RespectForBotTokens) != 0, + _ => throw new ArgumentOutOfRangeException(nameof(tokenKind)), + }; + + public static string GetDisplayName(this RateLimitPreference rateLimitPreference) => + rateLimitPreference switch + { + RateLimitPreference.IgnoreAll => "Always ignore", + RateLimitPreference.RespectForUserTokens => "Respect for user tokens", + RateLimitPreference.RespectForBotTokens => "Respect for bot tokens", + RateLimitPreference.RespectAll => "Always respect", + _ => throw new ArgumentOutOfRangeException(nameof(rateLimitPreference)), + }; +} diff --git a/DiscordChatExporter.Gui/Converters/RateLimitPreferenceToStringConverter.cs b/DiscordChatExporter.Gui/Converters/RateLimitPreferenceToStringConverter.cs new file mode 100644 index 00000000..b34a7132 --- /dev/null +++ b/DiscordChatExporter.Gui/Converters/RateLimitPreferenceToStringConverter.cs @@ -0,0 +1,28 @@ +using System; +using System.Globalization; +using Avalonia.Data.Converters; +using DiscordChatExporter.Core.Discord; + +namespace DiscordChatExporter.Gui.Converters; + +public class RateLimitPreferenceToStringConverter : IValueConverter +{ + public static RateLimitPreferenceToStringConverter Instance { get; } = new(); + + public object? Convert( + object? value, + Type targetType, + object? parameter, + CultureInfo culture + ) => + value is RateLimitPreference rateLimitPreference + ? rateLimitPreference.GetDisplayName() + : default; + + public object ConvertBack( + object? value, + Type targetType, + object? parameter, + CultureInfo culture + ) => throw new NotSupportedException(); +} diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index a22d30ca..87fb914b 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -3,6 +3,7 @@ using System.IO; using System.Text.Json.Serialization; using Cogwheel; using CommunityToolkit.Mvvm.ComponentModel; +using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Exporting; using DiscordChatExporter.Gui.Framework; using DiscordChatExporter.Gui.Models; @@ -28,6 +29,10 @@ public partial class SettingsService() [ObservableProperty] public partial bool IsTokenPersisted { get; set; } = true; + [ObservableProperty] + public partial RateLimitPreference RateLimitPreference { get; set; } = + RateLimitPreference.RespectAll; + [ObservableProperty] public partial ThreadInclusionMode ThreadInclusionMode { get; set; } diff --git a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs index 5d1bf649..ad459332 100644 --- a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs @@ -123,7 +123,7 @@ public partial class DashboardViewModel : ViewModelBase AvailableChannels = null; SelectedChannels.Clear(); - _discord = new DiscordClient(token); + _discord = new DiscordClient(token, _settingsService.RateLimitPreference); _settingsService.LastToken = token; var guilds = await _discord.GetUserGuildsAsync(); diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs index 66c28b91..ab88d62a 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Utils.Extensions; using DiscordChatExporter.Gui.Framework; using DiscordChatExporter.Gui.Models; @@ -42,6 +43,15 @@ public class SettingsViewModel : DialogViewModelBase set => _settingsService.IsTokenPersisted = value; } + public IReadOnlyList AvailableRateLimitPreferences { get; } = + Enum.GetValues(); + + public RateLimitPreference RateLimitPreference + { + get => _settingsService.RateLimitPreference; + set => _settingsService.RateLimitPreference = value; + } + public IReadOnlyList AvailableThreadInclusionModes { get; } = Enum.GetValues(); diff --git a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.axaml b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.axaml index eec678eb..a7a3c554 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.axaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.axaml @@ -54,6 +54,25 @@ + + + + + + + + + + + +