calvin.erfmann/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-02-09 19:32:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize fetching threads (#1125)

Browse Source
This commit is contained in:
Adam Slatinský
2023-09-03 20:18:49 +02:00
committed by GitHub
parent d24550f521
commit 9583e2684d
5 changed files with 73 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
View File

@@ -76,6 +76,8 @@ public class ExportAllCommand : ExportCommandBase
var thread in Discord.GetGuildThreadsAsync(
guild.Id,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)

2
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
View File

@@ -54,6 +54,8 @@ public class ExportGuildCommand : ExportCommandBase
var thread in Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)

2
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
View File

@@ -52,6 +52,8 @@ public class GetChannelsCommand : DiscordCommandBase
await Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
null,
null,
cancellationToken
)
)

58
DiscordChatExporter.Core/Discord/DiscordClient.cs
View File

@@ -277,6 +277,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildThreadsAsync(
Snowflake guildId,
bool includeArchived = false,
Snowflake? before = null,
Snowflake? after = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
@@ -286,17 +288,29 @@ public class DiscordClient
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
var channels = await GetGuildChannelsAsync(guildId, cancellationToken);
var filteredChannels = channels
// Categories cannot have threads
.Where(c => c.Kind != ChannelKind.GuildCategory)
// Voice channels cannot have threads
.Where(c => !c.Kind.IsVoice())
// Ordinary channel or forum channel without LastMessageId cannot have threads
.Where(c => c.LastMessageId != null)
// Ff --before is specified, skip channels created after the specified date
.Where(c => before == null || before > c.Id);
// User accounts can only fetch threads using the search endpoint
if (tokenKind == TokenKind.User)
{
// Active threads
foreach (var channel in channels)
foreach (var channel in filteredChannels)
{
var currentOffset = 0;
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "false")
.SetQueryParameter("offset", currentOffset.ToString())
.Build();
@@ -306,14 +320,29 @@ public class DiscordClient
if (response is null)
break;
var containsOlder = false;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
var thread = Channel.Parse(threadJson, channel);
// if --after is specified, we can break early, because the threads are sorted by last message time
if (after is not null && after > thread.LastMessageId)
{
containsOlder = true;
break;
}
yield return thread;
currentOffset++;
}
if (containsOlder)
break;
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
@@ -322,13 +351,15 @@ public class DiscordClient
// Archived threads
if (includeArchived)
{
foreach (var channel in channels)
foreach (var channel in filteredChannels)
{
var currentOffset = 0;
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "true")
.SetQueryParameter("offset", currentOffset.ToString())
.Build();
@@ -338,14 +369,29 @@ public class DiscordClient
if (response is null)
break;
var containsOlder = false;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
var thread = Channel.Parse(threadJson, channel);
// if --after is specified, we can break early, because the threads are sorted by last message time
if (after is not null && after > thread.LastMessageId)
{
containsOlder = true;
break;
}
yield return thread;
currentOffset++;
}
if (containsOlder)
break;
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
@@ -357,7 +403,7 @@ public class DiscordClient
{
// Active threads
{
var parentsById = channels.ToDictionary(c => c.Id);
var parentsById = filteredChannels.ToDictionary(c => c.Id);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/threads/active",
@@ -379,7 +425,7 @@ public class DiscordClient
// Archived threads
if (includeArchived)
{
foreach (var channel in channels)
foreach (var channel in filteredChannels)
{
// Public archived threads
{

15
DiscordChatExporter.Core/Exporting/ChannelExporter.cs
View File

@@ -2,6 +2,7 @@
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using Gress;
@@ -33,6 +34,20 @@ public class ChannelExporter
);
}
// Check if the 'before' boundary is valid
if (request.Before is not null && request.Channel.Id > request.Before)
{
throw new DiscordChatExporterException(
"Channel does not contain any messages within the specified period."
);
}
// Skip forum channels, they are exported as threads
if (request.Channel.Kind == ChannelKind.GuildForum)
{
throw new DiscordChatExporterException("Channel is a forum.");
}
// Build context
var context = new ExportContext(_discord, request);
await context.PopulateChannelsAndRolesAsync(cancellationToken);