From c4bfb3424e691c4beb69400881164c54fefd2d90 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 21:14:57 +0200 Subject: [PATCH] Add option to reverse message order in exports (#1487) Co-authored-by: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .../Specs/HtmlContentSpecs.cs | 44 +++++++- .../Specs/JsonContentSpecs.cs | 47 +++++++- .../Commands/Base/ExportCommandBase.cs | 7 ++ .../Discord/DiscordClient.cs | 102 ++++++++++++++++++ .../Exporting/ChannelExporter.cs | 14 ++- .../Exporting/ExportRequest.cs | 4 + .../Exporting/HtmlMessageWriter.cs | 1 + .../Exporting/PlainTextMessageExtensions.cs | 6 +- .../LocalizationManager.English.cs | 3 + .../LocalizationManager.French.cs | 3 + .../LocalizationManager.German.cs | 3 + .../LocalizationManager.Spanish.cs | 3 + .../LocalizationManager.Ukrainian.cs | 3 + .../Localization/LocalizationManager.cs | 2 + .../Services/SettingsService.cs | 3 + .../Components/DashboardViewModel.cs | 1 + .../Dialogs/ExportSetupViewModel.cs | 8 +- .../Views/Dialogs/ExportSetupView.axaml | 9 ++ 18 files changed, 253 insertions(+), 10 deletions(-) diff --git a/DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs index fcb1356a..601cfdd3 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs @@ -1,7 +1,12 @@ -using System.Linq; +using System.IO; +using System.Linq; using System.Threading.Tasks; using AngleSharp.Dom; +using CliFx.Infrastructure; +using DiscordChatExporter.Cli.Commands; using DiscordChatExporter.Cli.Tests.Infra; +using DiscordChatExporter.Cli.Tests.Utils; +using DiscordChatExporter.Core.Exporting; using FluentAssertions; using Xunit; @@ -44,4 +49,41 @@ public class HtmlContentSpecs "Yeet" ); } + + [Fact] + public async Task I_can_export_a_channel_in_the_HTML_format_in_the_reverse_order() + { + // Arrange + using var file = TempFile.Create(); + + // Act + await new ExportChannelsCommand + { + Token = Secrets.DiscordToken, + ChannelIds = [ChannelIds.DateRangeTestCases], + ExportFormat = ExportFormat.HtmlDark, + OutputPath = file.Path, + Locale = "en-US", + IsUtcNormalizationEnabled = true, + IsReverseMessageOrder = true, + }.ExecuteAsync(new FakeConsole()); + + var document = Html.Parse(await File.ReadAllTextAsync(file.Path)); + var messages = document.QuerySelectorAll("[data-message-id]").ToArray(); + + // Assert + messages + .Select(e => e.GetAttribute("data-message-id")) + .Should() + .Equal( + "885169254029213696", + "868505973294268457", + "868505969821364245", + "868505966528835604", + "868490009366396958", + "866732113319428096", + "866710679758045195", + "866674314627121232" + ); + } } diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs index 0acc5f63..ec1e7f1f 100644 --- a/DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs +++ b/DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs @@ -1,7 +1,13 @@ -using System.Linq; +using System.IO; +using System.Linq; using System.Threading.Tasks; +using CliFx.Infrastructure; +using DiscordChatExporter.Cli.Commands; using DiscordChatExporter.Cli.Tests.Infra; +using DiscordChatExporter.Cli.Tests.Utils; +using DiscordChatExporter.Core.Exporting; using FluentAssertions; +using JsonExtensions; using Xunit; namespace DiscordChatExporter.Cli.Tests.Specs; @@ -43,4 +49,43 @@ public class JsonContentSpecs "Yeet" ); } + + [Fact] + public async Task I_can_export_a_channel_in_the_JSON_format_in_the_reverse_order() + { + // Arrange + using var file = TempFile.Create(); + + // Act + await new ExportChannelsCommand + { + Token = Secrets.DiscordToken, + ChannelIds = [ChannelIds.DateRangeTestCases], + ExportFormat = ExportFormat.Json, + OutputPath = file.Path, + Locale = "en-US", + IsUtcNormalizationEnabled = true, + IsReverseMessageOrder = true, + }.ExecuteAsync(new FakeConsole()); + + var messages = Json.Parse(await File.ReadAllTextAsync(file.Path)) + .GetProperty("messages") + .EnumerateArray() + .ToArray(); + + // Assert + messages + .Select(j => j.GetProperty("id").GetString()) + .Should() + .Equal( + "885169254029213696", + "868505973294268457", + "868505969821364245", + "868505966528835604", + "868490009366396958", + "866732113319428096", + "866710679758045195", + "866674314627121232" + ); + } } diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index 09ab2691..070c8e88 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -83,6 +83,12 @@ public abstract class ExportCommandBase : DiscordCommandBase )] public int ParallelLimit { get; init; } = 1; + [CommandOption( + "reverse", + Description = "Export messages in reverse chronological order (newest first)." + )] + public bool IsReverseMessageOrder { get; init; } + [CommandOption( "markdown", Description = "Process markdown, mentions, and other special tokens." @@ -267,6 +273,7 @@ public abstract class ExportCommandBase : DiscordCommandBase Before, PartitionLimit, MessageFilter, + IsReverseMessageOrder, ShouldFormatMarkdown, ShouldDownloadAssets, ShouldReuseAssets, diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs index e076a97e..c5ca8393 100644 --- a/DiscordChatExporter.Core/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs @@ -578,6 +578,28 @@ public class DiscordClient( } } + private async ValueTask TryGetFirstMessageAsync( + Snowflake channelId, + Snowflake? after = null, + Snowflake? before = null, + CancellationToken cancellationToken = default + ) + { + var url = new UrlBuilder() + .SetPath($"channels/{channelId}/messages") + .SetQueryParameter("limit", "1") + .SetQueryParameter("after", (after ?? Snowflake.Zero).ToString()) + .Build(); + + var response = await GetJsonResponseAsync(url, cancellationToken); + var message = response.EnumerateArray().Select(Message.Parse).FirstOrDefault(); + + if (message is null || before is not null && message.Timestamp > before.Value.ToDate()) + return null; + + return message; + } + private async ValueTask TryGetLastMessageAsync( Snowflake channelId, Snowflake? before = null, @@ -684,6 +706,86 @@ public class DiscordClient( } } + public async IAsyncEnumerable GetMessagesInReverseAsync( + Snowflake channelId, + Snowflake? after = null, + Snowflake? before = null, + IProgress? progress = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + // Get the first message (oldest) in the range to use as the lower bound for + // progress calculation. + var firstMessage = await TryGetFirstMessageAsync( + channelId, + after, + before, + cancellationToken + ); + if (firstMessage is null) + yield break; + + // Keep track of the last message in range in order to calculate the progress + var lastMessage = default(Message); + var currentBefore = before; + while (true) + { + var url = new UrlBuilder() + .SetPath($"channels/{channelId}/messages") + .SetQueryParameter("limit", "100") + .SetQueryParameter("before", currentBefore?.ToString()) + .Build(); + + var response = await GetJsonResponseAsync(url, cancellationToken); + + var messages = response.EnumerateArray().Select(Message.Parse).ToArray(); + + // Break if there are no messages (can happen if messages are deleted during execution) + if (!messages.Any()) + yield break; + + // If all messages are empty, make sure that it's not because the bot account doesn't + // have the Message Content Intent enabled. + // https://github.com/Tyrrrz/DiscordChatExporter/issues/1106#issuecomment-1741548959 + if ( + messages.All(m => m.IsEmpty) + && await ResolveTokenKindAsync(cancellationToken) == TokenKind.Bot + ) + { + var application = await GetApplicationAsync(cancellationToken); + if (!application.IsMessageContentIntentEnabled) + { + throw new DiscordChatExporterException( + "Provided bot account does not have the Message Content Intent enabled.", + true + ); + } + } + + foreach (var message in messages) + { + lastMessage ??= message; + + // Report progress based on timestamps + if (progress is not null) + { + var exportedDuration = (lastMessage.Timestamp - message.Timestamp).Duration(); + var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration(); + + progress.Report( + Percentage.FromFraction( + totalDuration > TimeSpan.Zero ? exportedDuration / totalDuration : 1 + ) + ); + } + + yield return message; + } + + currentBefore = messages.Last().Id; + } + } + public async IAsyncEnumerable GetMessageReactionsAsync( Snowflake channelId, Snowflake messageId, diff --git a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs index 68ca75b4..7e3c400c 100644 --- a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs +++ b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs @@ -64,15 +64,23 @@ public class ChannelExporter(DiscordClient discord) ); } - await foreach ( - var message in discord.GetMessagesAsync( + var messages = !request.IsReverseMessageOrder + ? discord.GetMessagesAsync( request.Channel.Id, request.After, request.Before, progress, cancellationToken ) - ) + : discord.GetMessagesInReverseAsync( + request.Channel.Id, + request.After, + request.Before, + progress, + cancellationToken + ); + + await foreach (var message in messages) { try { diff --git a/DiscordChatExporter.Core/Exporting/ExportRequest.cs b/DiscordChatExporter.Core/Exporting/ExportRequest.cs index c814df8a..aae3a921 100644 --- a/DiscordChatExporter.Core/Exporting/ExportRequest.cs +++ b/DiscordChatExporter.Core/Exporting/ExportRequest.cs @@ -33,6 +33,8 @@ public partial class ExportRequest public MessageFilter MessageFilter { get; } + public bool IsReverseMessageOrder { get; } + public bool ShouldFormatMarkdown { get; } public bool ShouldDownloadAssets { get; } @@ -55,6 +57,7 @@ public partial class ExportRequest Snowflake? before, PartitionLimit partitionLimit, MessageFilter messageFilter, + bool isReverseMessageOrder, bool shouldFormatMarkdown, bool shouldDownloadAssets, bool shouldReuseAssets, @@ -69,6 +72,7 @@ public partial class ExportRequest Before = before; PartitionLimit = partitionLimit; MessageFilter = messageFilter; + IsReverseMessageOrder = isReverseMessageOrder; ShouldFormatMarkdown = shouldFormatMarkdown; ShouldDownloadAssets = shouldDownloadAssets; ShouldReuseAssets = shouldReuseAssets; diff --git a/DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs b/DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs index 05b1bba2..e626c379 100644 --- a/DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs +++ b/DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs @@ -17,6 +17,7 @@ internal class HtmlMessageWriter(Stream stream, ExportContext context, string th private readonly HtmlMinifier _minifier = new(); private readonly List _messageGroup = []; + // Note: in reverse order, last message is earlier than first message private bool CanJoinGroup(Message message) { // If the group is empty, any message can join it diff --git a/DiscordChatExporter.Core/Exporting/PlainTextMessageExtensions.cs b/DiscordChatExporter.Core/Exporting/PlainTextMessageExtensions.cs index b8c89896..69a81d49 100644 --- a/DiscordChatExporter.Core/Exporting/PlainTextMessageExtensions.cs +++ b/DiscordChatExporter.Core/Exporting/PlainTextMessageExtensions.cs @@ -23,13 +23,11 @@ internal static class PlainTextMessageExtensions : "Removed a recipient.", MessageKind.Call => - $"Started a call that lasted { - message + $"Started a call that lasted {message .CallEndedTimestamp? .Pipe(t => t - message.Timestamp) .Pipe(t => t.TotalMinutes) - .ToString("n0", CultureInfo.InvariantCulture) ?? "0" - } minutes.", + .ToString("n0", CultureInfo.InvariantCulture) ?? "0"} minutes.", MessageKind.ChannelNameChange => !string.IsNullOrWhiteSpace(message.Content) ? $"Changed the channel name: {message.Content}" diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.English.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.English.cs index 9cb79162..7f7fc479 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.English.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.English.cs @@ -104,6 +104,9 @@ public partial class LocalizationManager [nameof(MessageFilterLabel)] = "Message filter", [nameof(MessageFilterTooltip)] = "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image'). See the documentation for more info.", + [nameof(ReverseMessageOrderLabel)] = "Reverse messages", + [nameof(ReverseMessageOrderTooltip)] = + "Export messages in reverse chronological order (newest first)", [nameof(FormatMarkdownLabel)] = "Format markdown", [nameof(FormatMarkdownTooltip)] = "Process markdown, mentions, and other special tokens", diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.French.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.French.cs index 3a61de17..c6ffe41f 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.French.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.French.cs @@ -106,6 +106,9 @@ public partial class LocalizationManager [nameof(MessageFilterLabel)] = "Filtre de messages", [nameof(MessageFilterTooltip)] = "Inclure uniquement les messages satisfaisant ce filtre (ex. 'from:foo#1234' ou 'has:image'). Voir la documentation pour plus d'informations.", + [nameof(ReverseMessageOrderLabel)] = "Inverser l'ordre des messages", + [nameof(ReverseMessageOrderTooltip)] = + "Exporter les messages en ordre chronologique inversé (les plus récents en premier)", [nameof(FormatMarkdownLabel)] = "Formater le markdown", [nameof(FormatMarkdownTooltip)] = "Traiter le markdown, les mentions et autres tokens spéciaux", diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.German.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.German.cs index fd3f1439..05eec130 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.German.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.German.cs @@ -110,6 +110,9 @@ public partial class LocalizationManager [nameof(MessageFilterLabel)] = "Nachrichtenfilter", [nameof(MessageFilterTooltip)] = "Nur Nachrichten einschließen, die diesem Filter entsprechen (z. B. 'from:foo#1234' oder 'has:image'). Weitere Informationen finden Sie in der Dokumentation.", + [nameof(ReverseMessageOrderLabel)] = "Nachrichtenreihenfolge umkehren", + [nameof(ReverseMessageOrderTooltip)] = + "Nachrichten in umgekehrter chronologischer Reihenfolge exportieren (neueste zuerst)", [nameof(FormatMarkdownLabel)] = "Markdown formatieren", [nameof(FormatMarkdownTooltip)] = "Markdown, Erwähnungen und andere spezielle Token verarbeiten", diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.Spanish.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.Spanish.cs index e4d8e46d..61912430 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.Spanish.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.Spanish.cs @@ -104,6 +104,9 @@ public partial class LocalizationManager [nameof(MessageFilterLabel)] = "Filtro de mensajes", [nameof(MessageFilterTooltip)] = "Solo incluir mensajes que satisfagan este filtro (p. ej. 'from:foo#1234' o 'has:image'). Consulte la documentación para más información.", + [nameof(ReverseMessageOrderLabel)] = "Invertir orden de mensajes", + [nameof(ReverseMessageOrderTooltip)] = + "Exportar mensajes en orden cronológico inverso (los más recientes primero)", [nameof(FormatMarkdownLabel)] = "Formatear markdown", [nameof(FormatMarkdownTooltip)] = "Procesar markdown, menciones y otros tokens especiales", diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.Ukrainian.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.Ukrainian.cs index 068cf8f9..b3ce2f48 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.Ukrainian.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.Ukrainian.cs @@ -104,6 +104,9 @@ public partial class LocalizationManager [nameof(MessageFilterLabel)] = "Фільтр повідомлень", [nameof(MessageFilterTooltip)] = "Включати лише повідомлення, що відповідають цьому фільтру (напр. 'from:foo#1234' або 'has:image'). Дивіться документацію для більш детальної інформації.", + [nameof(ReverseMessageOrderLabel)] = "Зворотній порядок повідомлень", + [nameof(ReverseMessageOrderTooltip)] = + "Експортувати повідомлення у зворотному хронологічному порядку (найновіші спочатку)", [nameof(FormatMarkdownLabel)] = "Форматувати markdown", [nameof(FormatMarkdownTooltip)] = "Обробляти markdown, згадки та інші спеціальні токени", diff --git a/DiscordChatExporter.Gui/Localization/LocalizationManager.cs b/DiscordChatExporter.Gui/Localization/LocalizationManager.cs index 7ab62e9f..71974f21 100644 --- a/DiscordChatExporter.Gui/Localization/LocalizationManager.cs +++ b/DiscordChatExporter.Gui/Localization/LocalizationManager.cs @@ -135,6 +135,8 @@ public partial class LocalizationManager public string PartitionLimitTooltip => Get(); public string MessageFilterLabel => Get(); public string MessageFilterTooltip => Get(); + public string ReverseMessageOrderLabel => Get(); + public string ReverseMessageOrderTooltip => Get(); public string FormatMarkdownLabel => Get(); public string FormatMarkdownTooltip => Get(); public string DownloadAssetsLabel => Get(); diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index cca8a3aa..f2a06709 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -61,6 +61,9 @@ public partial class SettingsService() [ObservableProperty] public partial string? LastMessageFilterValue { get; set; } + [ObservableProperty] + public partial bool LastIsReverseMessageOrder { get; set; } + [ObservableProperty] public partial bool LastShouldFormatMarkdown { get; set; } = true; diff --git a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs index ecbead8f..71e9429f 100644 --- a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs @@ -275,6 +275,7 @@ public partial class DashboardViewModel : ViewModelBase dialog.Before?.Pipe(Snowflake.FromDate), dialog.PartitionLimit, dialog.MessageFilter, + dialog.IsReverseMessageOrder, dialog.ShouldFormatMarkdown, dialog.ShouldDownloadAssets, dialog.ShouldReuseAssets, diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs index afee2f00..a5e57705 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs @@ -62,6 +62,9 @@ public partial class ExportSetupViewModel( [NotifyPropertyChangedFor(nameof(MessageFilter))] public partial string? MessageFilterValue { get; set; } + [ObservableProperty] + public partial bool IsReverseMessageOrder { get; set; } + [ObservableProperty] public partial bool ShouldFormatMarkdown { get; set; } @@ -106,6 +109,7 @@ public partial class ExportSetupViewModel( SelectedFormat = settingsService.LastExportFormat; PartitionLimitValue = settingsService.LastPartitionLimitValue; MessageFilterValue = settingsService.LastMessageFilterValue; + IsReverseMessageOrder = settingsService.LastIsReverseMessageOrder; ShouldFormatMarkdown = settingsService.LastShouldFormatMarkdown; ShouldDownloadAssets = settingsService.LastShouldDownloadAssets; ShouldReuseAssets = settingsService.LastShouldReuseAssets; @@ -120,7 +124,8 @@ public partial class ExportSetupViewModel( || !string.IsNullOrWhiteSpace(MessageFilterValue) || ShouldDownloadAssets || ShouldReuseAssets - || !string.IsNullOrWhiteSpace(AssetsDirPath); + || !string.IsNullOrWhiteSpace(AssetsDirPath) + || IsReverseMessageOrder; } [RelayCommand] @@ -184,6 +189,7 @@ public partial class ExportSetupViewModel( settingsService.LastExportFormat = SelectedFormat; settingsService.LastPartitionLimitValue = PartitionLimitValue; settingsService.LastMessageFilterValue = MessageFilterValue; + settingsService.LastIsReverseMessageOrder = IsReverseMessageOrder; settingsService.LastShouldFormatMarkdown = ShouldFormatMarkdown; settingsService.LastShouldDownloadAssets = ShouldDownloadAssets; settingsService.LastShouldReuseAssets = ShouldReuseAssets; diff --git a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.axaml b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.axaml index 5e288708..ca501f47 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.axaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.axaml @@ -196,6 +196,15 @@ Theme="{DynamicResource FilledTextBox}" ToolTip.Tip="{Binding LocalizationManager.MessageFilterTooltip}" /> + + + + + +