calvin.erfmann/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-01-28 22:01:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use CSharpier

Browse Source
This commit is contained in:
Tyrrrz
2023-08-22 21:17:19 +03:00
parent c410e745b1
commit 20f58963a6
174 changed files with 11084 additions and 10670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DiscordChatExporter.Cli.Tests/DiscordChatExporter.Cli.Tests.csproj
View File

@@ -8,11 +8,11 @@
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
<None Include="*.secret" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.0.4" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="FluentAssertions" Version="6.11.0" />
<PackageReference Include="GitHubActionsTestLogger" Version="2.3.2" PrivateAssets="all" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />

2
DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs
View File

@@ -23,4 +23,4 @@ public static class ChannelIds
public static Snowflake SelfContainedTestCases { get; } = Snowflake.Parse("887441432678379560");
public static Snowflake StickerTestCases { get; } = Snowflake.Parse("939668868253769729");
}
}

81
DiscordChatExporter.Cli.Tests/Infra/ExportWrapper.cs
View File

@@ -19,14 +19,16 @@ namespace DiscordChatExporter.Cli.Tests.Infra;
public static class ExportWrapper
{
private static readonly AsyncKeyedLocker<string> Locker = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly AsyncKeyedLocker<string> Locker =
new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly string DirPath = Path.Combine(
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"ExportCache"
);
@@ -36,9 +38,7 @@ public static class ExportWrapper
{
Directory.Delete(DirPath, true);
}
catch (DirectoryNotFoundException)
{
}
catch (DirectoryNotFoundException) { }
Directory.CreateDirectory(DirPath);
}
@@ -66,13 +66,11 @@ public static class ExportWrapper
return await File.ReadAllTextAsync(filePath);
}
public static async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId) => Html.Parse(
await ExportAsync(channelId, ExportFormat.HtmlDark)
);
public static async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId) =>
Html.Parse(await ExportAsync(channelId, ExportFormat.HtmlDark));
public static async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId) => Json.Parse(
await ExportAsync(channelId, ExportFormat.Json)
);
public static async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId) =>
Json.Parse(await ExportAsync(channelId, ExportFormat.Json));
public static async ValueTask<string> ExportAsPlainTextAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.PlainText);
@@ -80,25 +78,26 @@ public static class ExportWrapper
public static async ValueTask<string> ExportAsCsvAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.Csv);
public static async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(Snowflake channelId) =>
(await ExportAsHtmlAsync(channelId))
.QuerySelectorAll("[data-message-id]")
.ToArray();
public static async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(
Snowflake channelId
) => (await ExportAsHtmlAsync(channelId)).QuerySelectorAll("[data-message-id]").ToArray();
public static async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(Snowflake channelId) =>
(await ExportAsJsonAsync(channelId))
.GetProperty("messages")
.EnumerateArray()
.ToArray();
public static async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(
Snowflake channelId
) => (await ExportAsJsonAsync(channelId)).GetProperty("messages").EnumerateArray().ToArray();
public static async ValueTask<IElement> GetMessageAsHtmlAsync(Snowflake channelId, Snowflake messageId)
public static async ValueTask<IElement> GetMessageAsHtmlAsync(
Snowflake channelId,
Snowflake messageId
)
{
var message = (await GetMessagesAsHtmlAsync(channelId)).SingleOrDefault(e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
var message = (await GetMessagesAsHtmlAsync(channelId)).SingleOrDefault(
e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message is null)
@@ -111,14 +110,18 @@ public static class ExportWrapper
return message;
}
public static async ValueTask<JsonElement> GetMessageAsJsonAsync(Snowflake channelId, Snowflake messageId)
public static async ValueTask<JsonElement> GetMessageAsJsonAsync(
Snowflake channelId,
Snowflake messageId
)
{
var message = (await GetMessagesAsJsonAsync(channelId)).SingleOrDefault(j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
var message = (await GetMessagesAsJsonAsync(channelId)).SingleOrDefault(
j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message.ValueKind == JsonValueKind.Undefined)
@@ -130,4 +133,4 @@ public static class ExportWrapper
return message;
}
}
}

6
DiscordChatExporter.Cli.Tests/Infra/Secrets.cs
View File

@@ -12,6 +12,6 @@ internal static class Secrets
.Build();
public static string DiscordToken =>
Configuration["DISCORD_TOKEN"] ??
throw new InvalidOperationException("Discord token not provided for tests.");
}
Configuration["DISCORD_TOKEN"]
?? throw new InvalidOperationException("Discord token not provided for tests.");
}

26
DiscordChatExporter.Cli.Tests/Specs/CsvContentSpecs.cs
View File

@@ -14,16 +14,18 @@ public class CsvContentSpecs
var document = await ExportWrapper.ExportAsCsvAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
document
.Should()
.ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}

116
DiscordChatExporter.Cli.Tests/Specs/DateRangeSpecs.cs
View File

@@ -34,8 +34,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -43,21 +42,28 @@ public class DateRangeSpecs
timestamps.All(t => t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero),
new DateTimeOffset(2021, 09, 08, 14, 26, 35, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero),
new DateTimeOffset(2021, 09, 08, 14, 26, 35, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
[Fact]
@@ -78,8 +84,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -87,19 +92,26 @@ public class DateRangeSpecs
timestamps.All(t => t < before).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 19, 13, 34, 18, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 15, 58, 48, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 17, 23, 58, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 19, 13, 34, 18, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 15, 58, 48, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 17, 23, 58, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
[Fact]
@@ -122,8 +134,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -131,19 +142,26 @@ public class DateRangeSpecs
timestamps.All(t => t < before && t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
}
}

17
DiscordChatExporter.Cli.Tests/Specs/FilterSpecs.cs
View File

@@ -32,8 +32,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -58,8 +57,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("author").GetProperty("name").GetString())
@@ -84,8 +82,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -110,8 +107,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -136,12 +132,11 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
.Should()
.ContainSingle("This has mention");
}
}
}

24
DiscordChatExporter.Cli.Tests/Specs/HtmlAttachmentSpecs.cs
View File

@@ -20,11 +20,7 @@ public class HtmlAttachmentSpecs
);
// Assert
message.Text().Should().ContainAll(
"Generic file attachment",
"Test.txt",
"11 bytes"
);
message.Text().Should().ContainAll("Generic file attachment", "Test.txt", "11 bytes");
message
.QuerySelectorAll("a")
@@ -71,9 +67,11 @@ public class HtmlAttachmentSpecs
message.Text().Should().Contain("Video attachment");
var videoUrl = message.QuerySelector("video source")?.GetAttribute("src");
videoUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
videoUrl
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
}
[Fact]
@@ -91,8 +89,10 @@ public class HtmlAttachmentSpecs
message.Text().Should().Contain("Audio attachment");
var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src");
audioUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
audioUrl
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
}
}
}

48
DiscordChatExporter.Cli.Tests/Specs/HtmlContentSpecs.cs
View File

@@ -16,26 +16,32 @@ public class HtmlContentSpecs
var messages = await ExportWrapper.GetMessagesAsHtmlAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(e => e.GetAttribute("data-message-id")).Should().Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages
.Select(e => e.GetAttribute("data-message-id"))
.Should()
.Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages.SelectMany(e => e.Text()).Should().ContainInOrder(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
messages
.SelectMany(e => e.Text())
.Should()
.ContainInOrder(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}

33
DiscordChatExporter.Cli.Tests/Specs/HtmlEmbedSpecs.cs
View File

@@ -21,15 +21,21 @@ public class HtmlEmbedSpecs
);
// Assert
message.Text().Should().ContainAll(
"Embed author",
"Embed title",
"Embed description",
"Field 1", "Value 1",
"Field 2", "Value 2",
"Field 3", "Value 3",
"Embed footer"
);
message
.Text()
.Should()
.ContainAll(
"Embed author",
"Embed title",
"Embed description",
"Field 1",
"Value 1",
"Field 2",
"Value 2",
"Field 3",
"Value 3",
"Embed footer"
);
}
[Fact]
@@ -83,7 +89,12 @@ public class HtmlEmbedSpecs
.QuerySelectorAll("source")
.Select(e => e.GetAttribute("src"))
.WhereNotNull()
.Where(s => s.EndsWith("i_am_currently_feeling_slight_displeasure_of_what_you_have_just_sent_lqrem.mp4"))
.Where(
s =>
s.EndsWith(
"i_am_currently_feeling_slight_displeasure_of_what_you_have_just_sent_lqrem.mp4"
)
)
.Should()
.ContainSingle();
}
@@ -193,4 +204,4 @@ public class HtmlEmbedSpecs
// Assert
message.Text().Should().Contain("DiscordChatExporter TestServer");
}
}
}

13
DiscordChatExporter.Cli.Tests/Specs/HtmlGroupingSpecs.cs
View File

@@ -32,8 +32,7 @@ public class HtmlGroupingSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var messageGroups = Html
.Parse(await File.ReadAllTextAsync(file.Path))
var messageGroups = Html.Parse(await File.ReadAllTextAsync(file.Path))
.QuerySelectorAll(".chatlog__message-group");
messageGroups.Should().HaveCount(2);
@@ -59,12 +58,6 @@ public class HtmlGroupingSpecs
.QuerySelectorAll(".chatlog__content")
.Select(e => e.Text())
.Should()
.ContainInOrder(
"Eleventh",
"Twelveth",
"Thirteenth",
"Fourteenth",
"Fifteenth"
);
.ContainInOrder("Eleventh", "Twelveth", "Thirteenth", "Fourteenth", "Fifteenth");
}
}
}

7
DiscordChatExporter.Cli.Tests/Specs/HtmlMarkdownSpecs.cs
View File

@@ -170,7 +170,10 @@ public class HtmlMarkdownSpecs
);
// Assert
message.Text().Should().Contain("Full long timestamp: Sunday, February 12, 2023 3:36 PM");
message
.Text()
.Should()
.Contain("Full long timestamp: Sunday, February 12, 2023 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
@@ -225,4 +228,4 @@ public class HtmlMarkdownSpecs
TimeZoneInfo.ClearCachedData();
}
}
}
}

2
DiscordChatExporter.Cli.Tests/Specs/HtmlMentionSpecs.cs
View File

@@ -61,4 +61,4 @@ public class HtmlMentionSpecs
// Assert
message.Text().Should().Contain("Role mention: @Role 1");
}
}
}

21
DiscordChatExporter.Cli.Tests/Specs/HtmlReplySpecs.cs
View File

@@ -36,9 +36,11 @@ public class HtmlReplySpecs
// Assert
message.Text().Should().Contain("reply to deleted");
message.QuerySelector(".chatlog__reply-link")?.Text().Should().Contain(
"Original message was deleted or could not be loaded."
);
message
.QuerySelector(".chatlog__reply-link")
?.Text()
.Should()
.Contain("Original message was deleted or could not be loaded.");
}
[Fact]
@@ -54,7 +56,11 @@ public class HtmlReplySpecs
// Assert
message.Text().Should().Contain("reply to attachment");
message.QuerySelector(".chatlog__reply-link")?.Text().Should().Contain("Click to see attachment");
message
.QuerySelector(".chatlog__reply-link")
?.Text()
.Should()
.Contain("Click to see attachment");
}
[Fact]
@@ -84,8 +90,11 @@ public class HtmlReplySpecs
);
// Assert
message.Text().Should().Contain("This is a test message from an announcement channel on another server");
message
.Text()
.Should()
.Contain("This is a test message from an announcement channel on another server");
message.Text().Should().Contain("SERVER");
message.QuerySelector(".chatlog__reply-link").Should().BeNull();
}
}
}

6
DiscordChatExporter.Cli.Tests/Specs/HtmlStickerSpecs.cs
View File

@@ -32,7 +32,9 @@ public class HtmlStickerSpecs
);
// Assert
var stickerUrl = message.QuerySelector("[title='Yikes'] [data-source]")?.GetAttribute("data-source");
var stickerUrl = message
.QuerySelector("[title='Yikes'] [data-source]")
?.GetAttribute("data-source");
stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
}

48
DiscordChatExporter.Cli.Tests/Specs/JsonAttachmentSpecs.cs
View File

@@ -24,9 +24,13 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("Test.txt");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(11);
}
@@ -46,9 +50,13 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("bird-thumbnail.png");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(466335);
}
@@ -68,10 +76,18 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP4_640_3MG.mp4");
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
attachments[0]
.GetProperty("fileName")
.GetString()
.Should()
.Be("file_example_MP4_640_3MG.mp4");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(3114374);
}
@@ -90,10 +106,14 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP3_1MG.mp3");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(1087849);
}
}
}

48
DiscordChatExporter.Cli.Tests/Specs/JsonContentSpecs.cs
View File

@@ -15,26 +15,32 @@ public class JsonContentSpecs
var messages = await ExportWrapper.GetMessagesAsJsonAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(j => j.GetProperty("id").GetString()).Should().Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages
.Select(j => j.GetProperty("id").GetString())
.Should()
.Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages.Select(j => j.GetProperty("content").GetString()).Should().Equal(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
messages
.Select(j => j.GetProperty("content").GetString())
.Should()
.Equal(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}

2
DiscordChatExporter.Cli.Tests/Specs/JsonEmbedSpecs.cs
View File

@@ -52,4 +52,4 @@ public class JsonEmbedSpecs
embedFields[2].GetProperty("value").GetString().Should().Be("Value 3");
embedFields[2].GetProperty("isInline").GetBoolean().Should().BeTrue();
}
}
}

14
DiscordChatExporter.Cli.Tests/Specs/JsonMentionSpecs.cs
View File

@@ -39,7 +39,11 @@ public class JsonMentionSpecs
);
// Assert
message.GetProperty("content").GetString().Should().Be("Text channel mention: #mention-tests");
message
.GetProperty("content")
.GetString()
.Should()
.Be("Text channel mention: #mention-tests");
}
[Fact]
@@ -52,7 +56,11 @@ public class JsonMentionSpecs
);
// Assert
message.GetProperty("content").GetString().Should().Be("Voice channel mention: #general [voice]");
message
.GetProperty("content")
.GetString()
.Should()
.Be("Voice channel mention: #general [voice]");
}
[Fact]
@@ -67,4 +75,4 @@ public class JsonMentionSpecs
// Assert
message.GetProperty("content").GetString().Should().Be("Role mention: @Role 1");
}
}
}

24
DiscordChatExporter.Cli.Tests/Specs/JsonStickerSpecs.cs
View File

@@ -19,15 +19,16 @@ public class JsonStickerSpecs
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
var sticker = message.GetProperty("stickers").EnumerateArray().Single();
sticker.GetProperty("id").GetString().Should().Be("904215665597120572");
sticker.GetProperty("name").GetString().Should().Be("rock");
sticker.GetProperty("format").GetString().Should().Be("Apng");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
sticker
.GetProperty("sourceUrl")
.GetString()
.Should()
.Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
}
[Fact]
@@ -40,14 +41,15 @@ public class JsonStickerSpecs
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
var sticker = message.GetProperty("stickers").EnumerateArray().Single();
sticker.GetProperty("id").GetString().Should().Be("816087132447178774");
sticker.GetProperty("name").GetString().Should().Be("Yikes");
sticker.GetProperty("format").GetString().Should().Be("Lottie");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
sticker
.GetProperty("sourceUrl")
.GetString()
.Should()
.Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
}

10
DiscordChatExporter.Cli.Tests/Specs/PartitioningSpecs.cs
View File

@@ -31,9 +31,7 @@ public class PartitioningSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Directory.EnumerateFiles(dir.Path, "output*")
.Should()
.HaveCount(3);
Directory.EnumerateFiles(dir.Path, "output*").Should().HaveCount(3);
}
[Fact]
@@ -54,8 +52,6 @@ public class PartitioningSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Directory.EnumerateFiles(dir.Path, "output*")
.Should()
.HaveCount(8);
Directory.EnumerateFiles(dir.Path, "output*").Should().HaveCount(8);
}
}
}

26
DiscordChatExporter.Cli.Tests/Specs/PlainTextContentSpecs.cs
View File

@@ -14,16 +14,18 @@ public class PlainTextContentSpecs
var document = await ExportWrapper.ExportAsPlainTextAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
document
.Should()
.ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}

5
DiscordChatExporter.Cli.Tests/Specs/SelfContainedSpecs.cs
View File

@@ -31,8 +31,7 @@ public class SelfContainedSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Html
.Parse(await File.ReadAllTextAsync(filePath))
Html.Parse(await File.ReadAllTextAsync(filePath))
.QuerySelectorAll("body [src]")
.Select(e => e.GetAttribute("src")!)
.Select(f => Path.GetFullPath(f, dir.Path))
@@ -40,4 +39,4 @@ public class SelfContainedSpecs
.Should()
.BeTrue();
}
}
}

2
DiscordChatExporter.Cli.Tests/Utils/Html.cs
View File

@@ -8,4 +8,4 @@ internal static class Html
private static readonly IHtmlParser Parser = new HtmlParser();
public static IHtmlDocument Parse(string source) => Parser.ParseDocument(source);
}
}

12
DiscordChatExporter.Cli.Tests/Utils/TempDir.cs
View File

@@ -9,8 +9,7 @@ internal partial class TempDir : IDisposable
{
public string Path { get; }
public TempDir(string path) =>
Path = path;
public TempDir(string path) => Path = path;
public void Dispose()
{
@@ -18,9 +17,7 @@ internal partial class TempDir : IDisposable
{
Directory.Delete(Path, true);
}
catch (DirectoryNotFoundException)
{
}
catch (DirectoryNotFoundException) { }
}
}
@@ -29,7 +26,8 @@ internal partial class TempDir
public static TempDir Create()
{
var dirPath = PathEx.Combine(
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"Temp",
Guid.NewGuid().ToString()
);
@@ -38,4 +36,4 @@ internal partial class TempDir
return new TempDir(dirPath);
}
}
}

17
DiscordChatExporter.Cli.Tests/Utils/TempFile.cs
View File

@@ -9,8 +9,7 @@ internal partial class TempFile : IDisposable
{
public string Path { get; }
public TempFile(string path) =>
Path = path;
public TempFile(string path) => Path = path;
public void Dispose()
{
@@ -18,9 +17,7 @@ internal partial class TempFile : IDisposable
{
File.Delete(Path);
}
catch (FileNotFoundException)
{
}
catch (FileNotFoundException) { }
}
}
@@ -29,17 +26,15 @@ internal partial class TempFile
public static TempFile Create()
{
var dirPath = PathEx.Combine(
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"Temp"
);
Directory.CreateDirectory(dirPath);
var filePath = PathEx.Combine(
dirPath,
Guid.NewGuid() + ".tmp"
);
var filePath = PathEx.Combine(dirPath, Guid.NewGuid() + ".tmp");
return new TempFile(filePath);
}
}
}

2
DiscordChatExporter.Cli.Tests/Utils/TimeZoneInfoEx.cs
View File

@@ -11,4 +11,4 @@ internal static class TimeZoneInfoEx
public static void SetLocal(TimeSpan offset) =>
SetLocal(TimeZoneInfo.CreateCustomTimeZone("test-tz", offset, "test-tz", "test-tz"));
}
}

8
DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs
View File

@@ -38,9 +38,9 @@ public abstract class DiscordCommandBase : ICommand
using (console.WithForegroundColor(ConsoleColor.DarkYellow))
{
console.Error.WriteLine(
"Warning: Option --bot is deprecated and should not be used. " +
"The type of the provided token is now inferred automatically. " +
"Please update your workflows as this option may be completely removed in a future version."
"Warning: Option --bot is deprecated and should not be used. "
+ "The type of the provided token is now inferred automatically. "
+ "Please update your workflows as this option may be completely removed in a future version."
);
}
}
@@ -48,4 +48,4 @@ public abstract class DiscordCommandBase : ICommand
return default;
}
}
}

197
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
View File

@@ -28,11 +28,10 @@ public abstract class ExportCommandBase : DiscordCommandBase
[CommandOption(
"output",
'o',
Description =
"Output file or directory path. " +
"Directory path must end with a slash to avoid ambiguity. " +
"If a directory is specified, file names will be generated automatically. " +
"Supports template tokens, see the documentation for more info."
Description = "Output file or directory path. "
+ "Directory path must end with a slash to avoid ambiguity. "
+ "If a directory is specified, file names will be generated automatically. "
+ "Supports template tokens, see the documentation for more info."
)]
public string OutputPath
{
@@ -42,11 +41,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
init => _outputPath = Path.GetFullPath(value);
}
[CommandOption(
"format",
'f',
Description = "Export format."
)]
[CommandOption("format", 'f', Description = "Export format.")]
public ExportFormat ExportFormat { get; init; } = ExportFormat.HtmlDark;
[CommandOption(
@@ -64,17 +59,15 @@ public abstract class ExportCommandBase : DiscordCommandBase
[CommandOption(
"partition",
'p',
Description =
"Split the output into partitions, each limited to the specified " +
"number of messages (e.g. '100') or file size (e.g. '10mb')."
Description = "Split the output into partitions, each limited to the specified "
+ "number of messages (e.g. '100') or file size (e.g. '10mb')."
)]
public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
[CommandOption(
"filter",
Description =
"Only include messages that satisfy this filter. " +
"See the documentation for more info."
Description = "Only include messages that satisfy this filter. "
+ "See the documentation for more info."
)]
public MessageFilter MessageFilter { get; init; } = MessageFilter.Null;
@@ -106,9 +99,8 @@ public abstract class ExportCommandBase : DiscordCommandBase
[CommandOption(
"media-dir",
Description =
"Download assets to this directory. " +
"If not specified, the asset directory path will be derived from the output path."
Description = "Download assets to this directory. "
+ "If not specified, the asset directory path will be derived from the output path."
)]
public string? AssetsDirPath
{
@@ -118,10 +110,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
init => _assetsDirPath = value is not null ? Path.GetFullPath(value) : null;
}
[CommandOption(
"dateformat",
Description = "Format used when writing dates."
)]
[CommandOption("dateformat", Description = "Format used when writing dates.")]
public string DateFormat { get; init; } = "MM/dd/yyyy h:mm tt";
[CommandOption(
@@ -142,17 +131,13 @@ public abstract class ExportCommandBase : DiscordCommandBase
// https://github.com/Tyrrrz/DiscordChatExporter/issues/425
if (ShouldReuseAssets && !ShouldDownloadAssets)
{
throw new CommandException(
"Option --reuse-media cannot be used without --media."
);
throw new CommandException("Option --reuse-media cannot be used without --media.");
}
// Assets directory can only be specified if the download assets option is set
if (!string.IsNullOrWhiteSpace(AssetsDirPath) && !ShouldDownloadAssets)
{
throw new CommandException(
"Option --media-dir cannot be used without --media."
);
throw new CommandException("Option --media-dir cannot be used without --media.");
}
// Make sure the user does not try to export multiple channels into one file.
@@ -161,17 +146,20 @@ public abstract class ExportCommandBase : DiscordCommandBase
// https://github.com/Tyrrrz/DiscordChatExporter/issues/917
var isValidOutputPath =
// Anything is valid when exporting a single channel
channels.Count <= 1 ||
channels.Count <= 1
||
// When using template tokens, assume the user knows what they're doing
OutputPath.Contains('%') ||
OutputPath.Contains('%')
||
// Otherwise, require an existing directory or an unambiguous directory path
Directory.Exists(OutputPath) || PathEx.IsDirectoryPath(OutputPath);
Directory.Exists(OutputPath)
|| PathEx.IsDirectoryPath(OutputPath);
if (!isValidOutputPath)
{
throw new CommandException(
"Attempted to export multiple channels, but the output path is neither a directory nor a template. " +
"If the provided output path is meant to be treated as a directory, make sure it ends with a slash."
"Attempted to export multiple channels, but the output path is neither a directory nor a template. "
+ "If the provided output path is meant to be treated as a directory, make sure it ends with a slash."
);
}
@@ -180,56 +168,61 @@ public abstract class ExportCommandBase : DiscordCommandBase
var errorsByChannel = new ConcurrentDictionary<Channel, string>();
await console.Output.WriteLineAsync($"Exporting {channels.Count} channel(s)...");
await console.CreateProgressTicker().StartAsync(async progressContext =>
{
await Parallel.ForEachAsync(
channels,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(1, ParallelLimit),
CancellationToken = cancellationToken
},
async (channel, innerCancellationToken) =>
{
try
await console
.CreateProgressTicker()
.StartAsync(async progressContext =>
{
await Parallel.ForEachAsync(
channels,
new ParallelOptions
{
await progressContext.StartTaskAsync(
$"{channel.Category} / {channel.Name}",
async progress =>
{
var guild = await Discord.GetGuildAsync(channel.GuildId, innerCancellationToken);
var request = new ExportRequest(
guild,
channel,
OutputPath,
AssetsDirPath,
ExportFormat,
After,
Before,
PartitionLimit,
MessageFilter,
ShouldFormatMarkdown,
ShouldDownloadAssets,
ShouldReuseAssets,
DateFormat
);
await Exporter.ExportChannelAsync(
request,
progress.ToPercentageBased(),
innerCancellationToken
);
}
);
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
MaxDegreeOfParallelism = Math.Max(1, ParallelLimit),
CancellationToken = cancellationToken
},
async (channel, innerCancellationToken) =>
{
errorsByChannel[channel] = ex.Message;
try
{
await progressContext.StartTaskAsync(
$"{channel.Category} / {channel.Name}",
async progress =>
{
var guild = await Discord.GetGuildAsync(
channel.GuildId,
innerCancellationToken
);
var request = new ExportRequest(
guild,
channel,
OutputPath,
AssetsDirPath,
ExportFormat,
After,
Before,
PartitionLimit,
MessageFilter,
ShouldFormatMarkdown,
ShouldDownloadAssets,
ShouldReuseAssets,
DateFormat
);
await Exporter.ExportChannelAsync(
request,
progress.ToPercentageBased(),
innerCancellationToken
);
}
);
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{
errorsByChannel[channel] = ex.Message;
}
}
}
);
});
);
});
// Print the result
using (console.WithForegroundColor(ConsoleColor.White))
@@ -285,8 +278,8 @@ public abstract class ExportCommandBase : DiscordCommandBase
if (channel.Kind == ChannelKind.GuildCategory)
{
var guildChannels =
channelsByGuild.GetValueOrDefault(channel.GuildId) ??
await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
channelsByGuild.GetValueOrDefault(channel.GuildId)
?? await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
foreach (var guildChannel in guildChannels)
{
@@ -311,18 +304,36 @@ public abstract class ExportCommandBase : DiscordCommandBase
// Support Ukraine callout
if (!IsUkraineSupportMessageDisabled)
{
console.Output.WriteLine("┌────────────────────────────────────────────────────────────────────┐");
console.Output.WriteLine("│ Thank you for supporting Ukraine <3 │");
console.Output.WriteLine("│ │");
console.Output.WriteLine("│ As Russia wages a genocidal war against my country, │");
console.Output.WriteLine("│ I'm grateful to everyone who continues to │");
console.Output.WriteLine("│ stand with Ukraine in our fight for freedom. │");
console.Output.WriteLine("│ │");
console.Output.WriteLine("│ Learn more: https://tyrrrz.me/ukraine │");
console.Output.WriteLine("└────────────────────────────────────────────────────────────────────┘");
console.Output.WriteLine(
"┌────────────────────────────────────────────────────────────────────┐"
);
console.Output.WriteLine(
"│ Thank you for supporting Ukraine <3 │"
);
console.Output.WriteLine(
"│ │"
);
console.Output.WriteLine(
"│ As Russia wages a genocidal war against my country, │"
);
console.Output.WriteLine(
"│ I'm grateful to everyone who continues to │"
);
console.Output.WriteLine(
"│ stand with Ukraine in our fight for freedom. │"
);
console.Output.WriteLine(
"│ │"
);
console.Output.WriteLine(
"│ Learn more: https://tyrrrz.me/ukraine │"
);
console.Output.WriteLine(
"└────────────────────────────────────────────────────────────────────┘"
);
console.Output.WriteLine("");
}
await base.ExecuteAsync(console);
}
}
}

2
DiscordChatExporter.Cli/Commands/Converters/TruthyBooleanBindingConverter.cs
View File

@@ -22,4 +22,4 @@ internal class TruthyBooleanBindingConverter : BindingConverter<bool>
return true;
}
}
}

52
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
View File

@@ -16,41 +16,25 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("exportall", Description = "Exports all accessible channels.")]
public class ExportAllCommand : ExportCommandBase
{
[CommandOption(
"include-dm",
Description = "Include direct message channels."
)]
[CommandOption("include-dm", Description = "Include direct message channels.")]
public bool IncludeDirectChannels { get; init; } = true;
[CommandOption(
"include-guilds",
Description = "Include guild channels."
)]
[CommandOption("include-guilds", Description = "Include guild channels.")]
public bool IncludeGuildChannels { get; init; } = true;
[CommandOption(
"include-vc",
Description = "Include voice channels."
)]
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Include threads."
)]
[CommandOption("include-threads", Description = "Include threads.")]
public bool IncludeThreads { get; init; } = false;
[CommandOption(
"include-archived-threads",
Description = "Include archived threads."
)]
[CommandOption("include-archived-threads", Description = "Include archived threads.")]
public bool IncludeArchivedThreads { get; init; } = false;
[CommandOption(
"data-package",
Description =
"Path to the personal data package (ZIP file) requested from Discord. " +
"If provided, only channels referenced in the dump will be exported."
Description = "Path to the personal data package (ZIP file) requested from Discord. "
+ "If provided, only channels referenced in the dump will be exported."
)]
public string? DataPackageFilePath { get; init; }
@@ -77,7 +61,9 @@ public class ExportAllCommand : ExportCommandBase
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
// Regular channels
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
await foreach (
var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)
)
{
if (channel.Kind == ChannelKind.GuildCategory)
continue;
@@ -91,7 +77,13 @@ public class ExportAllCommand : ExportCommandBase
// Threads
if (IncludeThreads)
{
await foreach (var thread in Discord.GetGuildThreadsAsync(guild.Id, IncludeArchivedThreads, cancellationToken))
await foreach (
var thread in Discord.GetGuildThreadsAsync(
guild.Id,
IncludeArchivedThreads,
cancellationToken
)
)
{
channels.Add(thread);
}
@@ -120,7 +112,9 @@ public class ExportAllCommand : ExportCommandBase
if (channelName is null)
continue;
await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})...");
await console.Output.WriteLineAsync(
$"Fetching channel '{channelName}' ({channelId})..."
);
try
{
@@ -129,7 +123,9 @@ public class ExportAllCommand : ExportCommandBase
}
catch (DiscordChatExporterException)
{
await console.Error.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible.");
await console.Error.WriteLineAsync(
$"Channel '{channelName}' ({channelId}) is inaccessible."
);
}
}
}
@@ -148,4 +144,4 @@ public class ExportAllCommand : ExportCommandBase
await ExportAsync(console, channels);
}
}
}

7
DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs
View File

@@ -14,9 +14,8 @@ public class ExportChannelsCommand : ExportCommandBase
[CommandOption(
"channel",
'c',
Description =
"Channel ID(s). " +
"If provided with category ID(s), all channels inside those categories will be exported."
Description = "Channel ID(s). "
+ "If provided with category ID(s), all channels inside those categories will be exported."
)]
public required IReadOnlyList<Snowflake> ChannelIds { get; init; }
@@ -25,4 +24,4 @@ public class ExportChannelsCommand : ExportCommandBase
await base.ExecuteAsync(console);
await ExportAsync(console, ChannelIds);
}
}
}

7
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
View File

@@ -17,8 +17,11 @@ public class ExportDirectMessagesCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var channels = await Discord.GetGuildChannelsAsync(
Guild.DirectMessages.Id,
cancellationToken
);
await ExportAsync(console, channels);
}
}
}

31
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
View File

@@ -12,29 +12,16 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("exportguild", Description = "Exports all channels within the specified guild.")]
public class ExportGuildCommand : ExportCommandBase
{
[CommandOption(
"guild",
'g',
Description = "Guild ID."
)]
[CommandOption("guild", 'g', Description = "Guild ID.")]
public required Snowflake GuildId { get; init; }
[CommandOption(
"include-vc",
Description = "Include voice channels."
)]
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Include threads."
)]
[CommandOption("include-threads", Description = "Include threads.")]
public bool IncludeThreads { get; init; } = false;
[CommandOption(
"include-archived-threads",
Description = "Include archived threads."
)]
[CommandOption("include-archived-threads", Description = "Include archived threads.")]
public bool IncludeArchivedThreads { get; init; } = false;
public override async ValueTask ExecuteAsync(IConsole console)
@@ -69,7 +56,13 @@ public class ExportGuildCommand : ExportCommandBase
// Threads
if (IncludeThreads)
{
await foreach (var thread in Discord.GetGuildThreadsAsync(GuildId, IncludeArchivedThreads, cancellationToken))
await foreach (
var thread in Discord.GetGuildThreadsAsync(
GuildId,
IncludeArchivedThreads,
cancellationToken
)
)
{
channels.Add(thread);
}
@@ -77,4 +70,4 @@ public class ExportGuildCommand : ExportCommandBase
await ExportAsync(console, channels);
}
}
}

35
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
View File

@@ -14,29 +14,16 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("channels", Description = "Get the list of channels in a guild.")]
public class GetChannelsCommand : DiscordCommandBase
{
[CommandOption(
"guild",
'g',
Description = "Guild ID."
)]
[CommandOption("guild", 'g', Description = "Guild ID.")]
public required Snowflake GuildId { get; init; }
[CommandOption(
"include-vc",
Description = "Include voice channels."
)]
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Include threads."
)]
[CommandOption("include-threads", Description = "Include threads.")]
public bool IncludeThreads { get; init; } = false;
[CommandOption(
"include-archived-threads",
Description = "Include archived threads."
)]
[CommandOption("include-archived-threads", Description = "Include archived threads.")]
public bool IncludeArchivedThreads { get; init; } = false;
public override async ValueTask ExecuteAsync(IConsole console)
@@ -66,7 +53,13 @@ public class GetChannelsCommand : DiscordCommandBase
.FirstOrDefault();
var threads = IncludeThreads
? (await Discord.GetGuildThreadsAsync(GuildId, IncludeArchivedThreads, cancellationToken))
? (
await Discord.GetGuildThreadsAsync(
GuildId,
IncludeArchivedThreads,
cancellationToken
)
)
.OrderBy(c => c.Name)
.ToArray()
: Array.Empty<Channel>();
@@ -116,8 +109,10 @@ public class GetChannelsCommand : DiscordCommandBase
// Thread status
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteLineAsync(channelThread.IsArchived ? "Archived" : "Active");
await console.Output.WriteLineAsync(
channelThread.IsArchived ? "Archived" : "Active"
);
}
}
}
}
}

6
DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs
View File

@@ -18,7 +18,9 @@ public class GetDirectChannelsCommand : DiscordCommandBase
var cancellationToken = console.RegisterCancellationHandler();
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
var channels = (
await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken)
)
.Where(c => c.Kind != ChannelKind.GuildCategory)
.OrderByDescending(c => c.LastMessageId)
.ThenBy(c => c.Name)
@@ -45,4 +47,4 @@ public class GetDirectChannelsCommand : DiscordCommandBase
await console.Output.WriteLineAsync($"{channel.Category} / {channel.Name}");
}
}
}
}

6
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
View File

@@ -32,9 +32,7 @@ public class GetGuildsCommand : DiscordCommandBase
foreach (var guild in guilds)
{
// Guild ID
await console.Output.WriteAsync(
guild.Id.ToString().PadRight(guildIdMaxLength, ' ')
);
await console.Output.WriteAsync(guild.Id.ToString().PadRight(guildIdMaxLength, ' '));
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
@@ -45,4 +43,4 @@ public class GetGuildsCommand : DiscordCommandBase
await console.Output.WriteLineAsync(guild.Name);
}
}
}
}

26
DiscordChatExporter.Cli/Commands/GuideCommand.cs
View File

@@ -15,14 +15,18 @@ public class GuideCommand : ICommand
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get user token:");
console.Output.WriteLine(" * Automating user accounts is technically against TOS — USE AT YOUR OWN RISK!");
console.Output.WriteLine(
" * Automating user accounts is technically against TOS — USE AT YOUR OWN RISK!"
);
console.Output.WriteLine(" 1. Open Discord in your web browser and login");
console.Output.WriteLine(" 2. Open any server or direct message channel");
console.Output.WriteLine(" 3. Press Ctrl+Shift+I to show developer tools");
console.Output.WriteLine(" 4. Navigate to the Network tab");
console.Output.WriteLine(" 5. Press Ctrl+R to reload");
console.Output.WriteLine(" 6. Switch between random channels to trigger network requests");
console.Output.WriteLine(" 7. Search for a request containing \"messages?limit=50\" or similar");
console.Output.WriteLine(
" 7. Search for a request containing \"messages?limit=50\" or similar"
);
console.Output.WriteLine(" 8. Select the Headers tab on the right");
console.Output.WriteLine(" 9. Scroll down to the Request Headers section");
console.Output.WriteLine(" 10. Copy the value of the \"authorization\" header");
@@ -36,7 +40,9 @@ public class GuideCommand : ICommand
console.Output.WriteLine(" 2. Open your application's settings");
console.Output.WriteLine(" 3. Navigate to the Bot section on the left");
console.Output.WriteLine(" 4. Under Token click Copy");
console.Output.WriteLine(" * Your bot needs to have Message Content Intent enabled to read messages");
console.Output.WriteLine(
" * Your bot needs to have Message Content Intent enabled to read messages"
);
console.Output.WriteLine();
// Guild or channel ID
@@ -47,15 +53,21 @@ public class GuideCommand : ICommand
console.Output.WriteLine(" 2. Open Settings");
console.Output.WriteLine(" 3. Go to Advanced section");
console.Output.WriteLine(" 4. Enable Developer Mode");
console.Output.WriteLine(" 5. Right-click on the desired guild or channel and click Copy Server ID or Copy Channel ID");
console.Output.WriteLine(
" 5. Right-click on the desired guild or channel and click Copy Server ID or Copy Channel ID"
);
console.Output.WriteLine();
// Docs link
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("If you have questions or issues, please refer to the documentation:");
console.Output.WriteLine(
"If you have questions or issues, please refer to the documentation:"
);
using (console.WithForegroundColor(ConsoleColor.DarkCyan))
console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs");
console.Output.WriteLine(
"https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs"
);
return default;
}
}
}

1
DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj
View File

@@ -7,6 +7,7 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="2.3.4" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Deorcify" Version="1.0.2" PrivateAssets="all" />
<PackageReference Include="DotnetRuntimeBootstrapper" Version="2.5.1" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />

5
DiscordChatExporter.Cli/Program.cs
View File

@@ -1,6 +1,3 @@
using CliFx;
return await new CliApplicationBuilder()
.AddCommandsFromThisAssembly()
.Build()
.RunAsync(args);
return await new CliApplicationBuilder().AddCommandsFromThisAssembly().Build().RunAsync(args);

44
DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs
View File

@@ -8,34 +8,38 @@ namespace DiscordChatExporter.Cli.Utils.Extensions;
internal static class ConsoleExtensions
{
public static IAnsiConsole CreateAnsiConsole(this IConsole console) =>
AnsiConsole.Create(new AnsiConsoleSettings
{
Ansi = AnsiSupport.Detect,
ColorSystem = ColorSystemSupport.Detect,
Out = new AnsiConsoleOutput(console.Output)
});
public static Progress CreateProgressTicker(this IConsole console) => console
.CreateAnsiConsole()
.Progress()
.AutoClear(false)
.AutoRefresh(true)
.HideCompleted(false)
.Columns(
new TaskDescriptionColumn {Alignment = Justify.Left},
new ProgressBarColumn(),
new PercentageColumn()
AnsiConsole.Create(
new AnsiConsoleSettings
{
Ansi = AnsiSupport.Detect,
ColorSystem = ColorSystemSupport.Detect,
Out = new AnsiConsoleOutput(console.Output)
}
);
public static Progress CreateProgressTicker(this IConsole console) =>
console
.CreateAnsiConsole()
.Progress()
.AutoClear(false)
.AutoRefresh(true)
.HideCompleted(false)
.Columns(
new TaskDescriptionColumn { Alignment = Justify.Left },
new ProgressBarColumn(),
new PercentageColumn()
);
public static async ValueTask StartTaskAsync(
this ProgressContext progressContext,
string description,
Func<ProgressTask, ValueTask> performOperationAsync)
Func<ProgressTask, ValueTask> performOperationAsync
)
{
var progressTask = progressContext.AddTask(
// Don't recognize random square brackets as style tags
Markup.Escape(description),
new ProgressTaskSettings {MaxValue = 1}
new ProgressTaskSettings { MaxValue = 1 }
);
try
@@ -48,4 +52,4 @@ internal static class ConsoleExtensions
progressTask.StopTask();
}
}
}
}

35
DiscordChatExporter.Core/Discord/Data/Attachment.cs
View File

@@ -15,30 +15,31 @@ public partial record Attachment(
string? Description,
int? Width,
int? Height,
FileSize FileSize) : IHasId
FileSize FileSize
) : IHasId
{
public string FileExtension => Path.GetExtension(FileName);
public bool IsImage =>
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
public bool IsVideo =>
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
public bool IsAudio =>
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
public bool IsSpoiler => FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
}
@@ -57,4 +58,4 @@ public partial record Attachment
return new Attachment(id, url, fileName, description, width, height, fileSize);
}
}
}

76
DiscordChatExporter.Core/Discord/Data/Channel.cs
View File

@@ -17,24 +17,27 @@ public partial record Channel(
string? IconUrl,
string? Topic,
bool IsArchived,
Snowflake? LastMessageId) : IHasId
Snowflake? LastMessageId
) : IHasId
{
// Used for visual backwards-compatibility with old exports, where
// channels without a parent (i.e. mostly DM channels) or channels
// with an inaccessible parent (i.e. inside private categories) had
// a fallback category created for them.
public string Category => Parent?.Name ?? Kind switch
{
ChannelKind.GuildCategory => "Category",
ChannelKind.GuildTextChat => "Text",
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildPrivateThread => "Private Thread",
ChannelKind.GuildPublicThread => "Public Thread",
ChannelKind.GuildNews => "News",
ChannelKind.GuildNewsThread => "News Thread",
_ => "Default"
};
public string Category =>
Parent?.Name
?? Kind switch
{
ChannelKind.GuildCategory => "Category",
ChannelKind.GuildTextChat => "Text",
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildPrivateThread => "Private Thread",
ChannelKind.GuildPublicThread => "Public Thread",
ChannelKind.GuildNews => "News",
ChannelKind.GuildNewsThread => "News Thread",
_ => "Default"
};
// Only needed for WPF data binding. Don't use anywhere else.
public bool IsVoice => Kind.IsVoice();
@@ -48,44 +51,41 @@ public partial record Channel
var kind = (ChannelKind)json.GetProperty("type").GetInt32();
var guildId =
json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse) ??
Guild.DirectMessages.Id;
json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse) ?? Guild.DirectMessages.Id;
var name =
// Guild channel
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ??
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull()
??
// DM channel
json.GetPropertyOrNull("recipients")?
.EnumerateArrayOrNull()?
.Select(User.Parse)
json.GetPropertyOrNull("recipients")
?.EnumerateArrayOrNull()
?.Select(User.Parse)
.Select(u => u.DisplayName)
.Pipe(s => string.Join(", ", s)) ??
.Pipe(s => string.Join(", ", s))
??
// Fallback
id.ToString();
var position =
positionHint ??
json.GetPropertyOrNull("position")?.GetInt32OrNull();
var position = positionHint ?? json.GetPropertyOrNull("position")?.GetInt32OrNull();
// Icons can only be set for group DM channels
var iconUrl = json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var iconUrl = json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var topic = json.GetPropertyOrNull("topic")?.GetStringOrNull();
var isArchived = json
.GetPropertyOrNull("thread_metadata")?
.GetPropertyOrNull("archived")?
.GetBooleanOrNull() ?? false;
var isArchived =
json.GetPropertyOrNull("thread_metadata")
?.GetPropertyOrNull("archived")
?.GetBooleanOrNull() ?? false;
var lastMessageId = json
.GetPropertyOrNull("last_message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var lastMessageId = json.GetPropertyOrNull("last_message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new Channel(
id,
@@ -100,4 +100,4 @@ public partial record Channel
lastMessageId
);
}
}
}

10
DiscordChatExporter.Core/Discord/Data/ChannelKind.cs
View File

@@ -22,12 +22,14 @@ public static class ChannelKindExtensions
public static bool IsDirect(this ChannelKind kind) =>
kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
public static bool IsGuild(this ChannelKind kind) =>
!kind.IsDirect();
public static bool IsGuild(this ChannelKind kind) => !kind.IsDirect();
public static bool IsVoice(this ChannelKind kind) =>
kind is ChannelKind.GuildVoiceChat or ChannelKind.GuildStageVoice;
public static bool IsThread(this ChannelKind kind) =>
kind is ChannelKind.GuildNewsThread or ChannelKind.GuildPublicThread or ChannelKind.GuildPrivateThread;
}
kind
is ChannelKind.GuildNewsThread
or ChannelKind.GuildPublicThread
or ChannelKind.GuildPrivateThread;
}

7
DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs
View File

@@ -41,10 +41,13 @@ public readonly partial record struct FileSize(long TotalBytes)
[ExcludeFromCodeCoverage]
public override string ToString() =>
string.Create(CultureInfo.InvariantCulture, $"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}");
string.Create(
CultureInfo.InvariantCulture,
$"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}"
);
}
public partial record struct FileSize
{
public static FileSize FromBytes(long bytes) => new(bytes);
}
}

2
DiscordChatExporter.Core/Discord/Data/Common/IHasId.cs
View File

@@ -3,4 +3,4 @@
public interface IHasId
{
Snowflake Id { get; }
}
}

14
DiscordChatExporter.Core/Discord/Data/Common/ImageCdn.cs
View File

@@ -19,10 +19,7 @@ public static class ImageCdn
? runes
: runes.Where(r => r.Value != 0xfe0f);
var twemojiId = string.Join(
"-",
filteredRunes.Select(r => r.Value.ToString("x"))
);
var twemojiId = string.Join("-", filteredRunes.Select(r => r.Value.ToString("x")));
return $"https://cdn.jsdelivr.net/gh/twitter/twemoji@latest/assets/svg/{twemojiId}.svg";
}
@@ -50,11 +47,16 @@ public static class ImageCdn
public static string GetFallbackUserAvatarUrl(int index = 0) =>
$"https://cdn.discordapp.com/embed/avatars/{index}.png";
public static string GetMemberAvatarUrl(Snowflake guildId, Snowflake userId, string avatarHash, int size = 512) =>
public static string GetMemberAvatarUrl(
Snowflake guildId,
Snowflake userId,
string avatarHash,
int size = 512
) =>
avatarHash.StartsWith("a_", StringComparison.Ordinal)
? $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.gif?size={size}"
: $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.png?size={size}";
public static string GetStickerUrl(Snowflake stickerId, string format = "png") =>
$"https://cdn.discordapp.com/stickers/{stickerId}.{format}";
}
}

28
DiscordChatExporter.Core/Discord/Data/Embeds/Embed.cs
View File

@@ -21,7 +21,8 @@ public partial record Embed(
EmbedImage? Thumbnail,
IReadOnlyList<EmbedImage> Images,
EmbedVideo? Video,
EmbedFooter? Footer)
EmbedFooter? Footer
)
{
// Embeds can only have one image according to the API model,
// but the client can render multiple images in some cases.
@@ -41,24 +42,25 @@ public partial record Embed
var title = json.GetPropertyOrNull("title")?.GetStringOrNull();
var kind =
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>() ??
EmbedKind.Rich;
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>()
?? EmbedKind.Rich;
var url = json.GetPropertyOrNull("url")?.GetNonWhiteSpaceStringOrNull();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffsetOrNull();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var description = json.GetPropertyOrNull("description")?.GetStringOrNull();
var fields =
json.GetPropertyOrNull("fields")?.EnumerateArrayOrNull()?.Select(EmbedField.Parse).ToArray() ??
Array.Empty<EmbedField>();
json.GetPropertyOrNull("fields")
?.EnumerateArrayOrNull()
?.Select(EmbedField.Parse)
.ToArray() ?? Array.Empty<EmbedField>();
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
@@ -70,8 +72,10 @@ public partial record Embed
// with this by merging related embeds at the end of the message parsing process.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
var images =
json.GetPropertyOrNull("image")?.Pipe(EmbedImage.Parse).ToSingletonEnumerable().ToArray() ??
Array.Empty<EmbedImage>();
json.GetPropertyOrNull("image")
?.Pipe(EmbedImage.Parse)
.ToSingletonEnumerable()
.ToArray() ?? Array.Empty<EmbedImage>();
var video = json.GetPropertyOrNull("video")?.Pipe(EmbedVideo.Parse);
@@ -92,4 +96,4 @@ public partial record Embed
footer
);
}
}
}

8
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedAuthor.cs
View File

@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure
public record EmbedAuthor(
string? Name,
string? Url,
string? IconUrl,
string? IconProxyUrl)
public record EmbedAuthor(string? Name, string? Url, string? IconUrl, string? IconProxyUrl)
{
public static EmbedAuthor Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedAuthor(
return new EmbedAuthor(name, url, iconUrl, iconProxyUrl);
}
}
}

7
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedField.cs
View File

@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure
public record EmbedField(
string Name,
string Value,
bool IsInline)
public record EmbedField(string Name, string Value, bool IsInline)
{
public static EmbedField Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedField(
return new EmbedField(name, value, isInline);
}
}
}

7
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedFooter.cs
View File

@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure
public record EmbedFooter(
string Text,
string? IconUrl,
string? IconProxyUrl)
public record EmbedFooter(string Text, string? IconUrl, string? IconProxyUrl)
{
public static EmbedFooter Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedFooter(
return new EmbedFooter(text, iconUrl, iconProxyUrl);
}
}
}

8
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedImage.cs
View File

@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure
public record EmbedImage(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedImage(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedImage Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedImage(
return new EmbedImage(url, proxyUrl, width, height);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedKind.cs
View File

@@ -8,4 +8,4 @@ public enum EmbedKind
Video,
Gifv,
Link
}
}

8
DiscordChatExporter.Core/Discord/Data/Embeds/EmbedVideo.cs
View File

@@ -4,11 +4,7 @@ using System.Text.Json;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-video-structure
public record EmbedVideo(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedVideo(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedVideo Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedVideo(
return new EmbedVideo(url, proxyUrl, width, height);
}
}
}

6
DiscordChatExporter.Core/Discord/Data/Embeds/SpotifyTrackEmbedProjection.cs
View File

@@ -12,7 +12,9 @@ public partial record SpotifyTrackEmbedProjection
private static string? TryParseTrackId(string embedUrl)
{
// https://open.spotify.com/track/1LHZMWefF9502NPfArRfvP?si=3efac6ce9be04f0a
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[1].Value;
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[
1
].Value;
if (!string.IsNullOrWhiteSpace(trackId))
return trackId;
@@ -33,4 +35,4 @@ public partial record SpotifyTrackEmbedProjection
return new SpotifyTrackEmbedProjection(trackId);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/Embeds/YouTubeVideoEmbedProjection.cs
View File

@@ -21,4 +21,4 @@ public partial record YouTubeVideoEmbedProjection
return new YouTubeVideoEmbedProjection(videoId);
}
}
}

23
DiscordChatExporter.Core/Discord/Data/Emoji.cs
View File

@@ -14,12 +14,11 @@ public partial record Emoji(
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated,
string ImageUrl)
string ImageUrl
)
{
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => Id is not null
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
}
public partial record Emoji
@@ -39,19 +38,17 @@ public partial record Emoji
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var id = json.GetPropertyOrNull("id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
// Names may be missing on custom emoji within reactions
var name = json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var name =
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(
id,
name,
isAnimated,
imageUrl
);
return new Emoji(id, name, isAnimated, imageUrl);
}
}
}

17710
DiscordChatExporter.Core/Discord/Data/EmojiIndex.cs
View File

File diff suppressed because it is too large Load Diff

17
DiscordChatExporter.Core/Discord/Data/Guild.cs
View File

@@ -9,11 +9,8 @@ namespace DiscordChatExporter.Core.Discord.Data;
public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
{
// Direct messages are encapsulated within a special pseudo-guild for consistency
public static Guild DirectMessages { get; } = new(
Snowflake.Zero,
"Direct Messages",
ImageCdn.GetFallbackUserAvatarUrl()
);
public static Guild DirectMessages { get; } =
new(Snowflake.Zero, "Direct Messages", ImageCdn.GetFallbackUserAvatarUrl());
public static Guild Parse(JsonElement json)
{
@@ -21,12 +18,10 @@ public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
var name = json.GetProperty("name").GetNonNullString();
var iconUrl =
json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl();
json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ?? ImageCdn.GetFallbackUserAvatarUrl();
return new Guild(id, name, iconUrl);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/Interaction.cs
View File

@@ -15,4 +15,4 @@ public record Interaction(Snowflake Id, string Name, User User)
return new Interaction(id, name, user);
}
}
}

7
DiscordChatExporter.Core/Discord/Data/Invite.cs
View File

@@ -6,10 +6,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/invite#invite-object
public record Invite(
string Code,
Guild Guild,
Channel? Channel)
public record Invite(string Code, Guild Guild, Channel? Channel)
{
public static string? TryGetCodeFromUrl(string url) =>
Regex.Match(url, @"^https?://discord\.gg/(\w+)/?$").Groups[1].Value.NullIfWhiteSpace();
@@ -22,4 +19,4 @@ public record Invite(
return new Invite(code, guild, channel);
}
}
}

31
DiscordChatExporter.Core/Discord/Data/Member.cs
View File

@@ -13,7 +13,8 @@ public partial record Member(
User User,
string? DisplayName,
string? AvatarUrl,
IReadOnlyList<Snowflake> RoleIds) : IHasId
IReadOnlyList<Snowflake> RoleIds
) : IHasId
{
public Snowflake Id => User.Id;
}
@@ -28,25 +29,19 @@ public partial record Member
var user = json.GetProperty("user").Pipe(User.Parse);
var displayName = json.GetPropertyOrNull("nick")?.GetNonWhiteSpaceStringOrNull();
var roleIds = json
.GetPropertyOrNull("roles")?
.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var roleIds =
json.GetPropertyOrNull("roles")
?.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var avatarUrl = guildId is not null
? json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
? json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
: null;
return new Member(
user,
displayName,
avatarUrl,
roleIds
);
return new Member(user, displayName, avatarUrl, roleIds);
}
}
}

68
DiscordChatExporter.Core/Discord/Data/Message.cs
View File

@@ -27,7 +27,8 @@ public partial record Message(
IReadOnlyList<User> MentionedUsers,
MessageReference? Reference,
Message? ReferencedMessage,
Interaction? Interaction) : IHasId
Interaction? Interaction
) : IHasId
{
public bool IsReplyLike => Kind == MessageKind.Reply || Interaction is not null;
@@ -70,22 +71,26 @@ public partial record Message
// Find embeds with the same URL that only contain a single image and nothing else
var trailingEmbeds = embeds
.Skip(i + 1)
.TakeWhile(e =>
e.Url == embed.Url &&
e.Timestamp is null &&
e.Author is null &&
e.Color is null &&
string.IsNullOrWhiteSpace(e.Description) &&
!e.Fields.Any() &&
e.Images.Count == 1 &&
e.Footer is null
.TakeWhile(
e =>
e.Url == embed.Url
&& e.Timestamp is null
&& e.Author is null
&& e.Color is null
&& string.IsNullOrWhiteSpace(e.Description)
&& !e.Fields.Any()
&& e.Images.Count == 1
&& e.Footer is null
)
.ToArray();
if (trailingEmbeds.Any())
{
// Concatenate all images into one embed
var images = embed.Images.Concat(trailingEmbeds.SelectMany(e => e.Images)).ToArray();
var images = embed.Images
.Concat(trailingEmbeds.SelectMany(e => e.Images))
.ToArray();
normalizedEmbeds.Add(embed with { Images = images });
i += trailingEmbeds.Length;
@@ -108,42 +113,49 @@ public partial record Message
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var kind = (MessageKind)json.GetProperty("type").GetInt32();
var flags = (MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var flags =
(MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var author = json.GetProperty("author").Pipe(User.Parse);
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json
.GetPropertyOrNull("call")?
.GetPropertyOrNull("ended_timestamp")?
.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json.GetPropertyOrNull("call")
?.GetPropertyOrNull("ended_timestamp")
?.GetDateTimeOffsetOrNull();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBooleanOrNull() ?? false;
var content = json.GetPropertyOrNull("content")?.GetStringOrNull() ?? "";
var attachments =
json.GetPropertyOrNull("attachments")?.EnumerateArrayOrNull()?.Select(Attachment.Parse).ToArray() ??
Array.Empty<Attachment>();
json.GetPropertyOrNull("attachments")
?.EnumerateArrayOrNull()
?.Select(Attachment.Parse)
.ToArray() ?? Array.Empty<Attachment>();
var embeds = NormalizeEmbeds(
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray() ??
Array.Empty<Embed>()
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray()
?? Array.Empty<Embed>()
);
var stickers =
json.GetPropertyOrNull("sticker_items")?.EnumerateArrayOrNull()?.Select(Sticker.Parse).ToArray() ??
Array.Empty<Sticker>();
json.GetPropertyOrNull("sticker_items")
?.EnumerateArrayOrNull()
?.Select(Sticker.Parse)
.ToArray() ?? Array.Empty<Sticker>();
var reactions =
json.GetPropertyOrNull("reactions")?.EnumerateArrayOrNull()?.Select(Reaction.Parse).ToArray() ??
Array.Empty<Reaction>();
json.GetPropertyOrNull("reactions")
?.EnumerateArrayOrNull()
?.Select(Reaction.Parse)
.ToArray() ?? Array.Empty<Reaction>();
var mentionedUsers =
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray() ??
Array.Empty<User>();
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray()
?? Array.Empty<User>();
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var messageReference = json.GetPropertyOrNull("message_reference")
?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var interaction = json.GetPropertyOrNull("interaction")?.Pipe(Interaction.Parse);
@@ -167,4 +179,4 @@ public partial record Message
interaction
);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/MessageFlags.cs
View File

@@ -15,4 +15,4 @@ public enum MessageFlags
HasThread = 32,
Ephemeral = 64,
Loading = 128
}
}

2
DiscordChatExporter.Core/Discord/Data/MessageKind.cs
View File

@@ -18,4 +18,4 @@ public enum MessageKind
public static class MessageKindExtensions
{
public static bool IsSystemNotification(this MessageKind kind) => (int)kind is >= 1 and <= 18;
}
}

23
DiscordChatExporter.Core/Discord/Data/MessageReference.cs
View File

@@ -9,21 +9,18 @@ public record MessageReference(Snowflake? MessageId, Snowflake? ChannelId, Snowf
{
public static MessageReference Parse(JsonElement json)
{
var messageId = json
.GetPropertyOrNull("message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var messageId = json.GetPropertyOrNull("message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var channelId = json
.GetPropertyOrNull("channel_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var channelId = json.GetPropertyOrNull("channel_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var guildId = json
.GetPropertyOrNull("guild_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new MessageReference(messageId, channelId, guildId);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/Reaction.cs
View File

@@ -13,4 +13,4 @@ public record Reaction(Emoji Emoji, int Count)
return new Reaction(emoji, count);
}
}
}

9
DiscordChatExporter.Core/Discord/Data/Role.cs
View File

@@ -15,13 +15,12 @@ public record Role(Snowflake Id, string Name, int Position, Color? Color) : IHas
var name = json.GetProperty("name").GetNonNullString();
var position = json.GetProperty("position").GetInt32();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha()
.NullIf(c => c.ToRgb() <= 0);
return new Role(id, name, position, color);
}
}
}

19
DiscordChatExporter.Core/Discord/Data/Sticker.cs
View File

@@ -15,14 +15,17 @@ public record Sticker(Snowflake Id, string Name, StickerFormat Format, string So
var name = json.GetProperty("name").GetNonNullString();
var format = (StickerFormat)json.GetProperty("format_type").GetInt32();
var sourceUrl = ImageCdn.GetStickerUrl(id, format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
});
var sourceUrl = ImageCdn.GetStickerUrl(
id,
format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
}
);
return new Sticker(id, name, format, sourceUrl);
}
}
}

2
DiscordChatExporter.Core/Discord/Data/StickerFormat.cs
View File

@@ -5,4 +5,4 @@ public enum StickerFormat
Png = 1,
Apng = 2,
Lottie = 3
}
}

33
DiscordChatExporter.Core/Discord/Data/User.cs
View File

@@ -15,18 +15,16 @@ public partial record User(
int? Discriminator,
string Name,
string DisplayName,
string AvatarUrl) : IHasId
string AvatarUrl
) : IHasId
{
public string DiscriminatorFormatted => Discriminator is not null
? $"{Discriminator:0000}"
: "0000";
public string DiscriminatorFormatted =>
Discriminator is not null ? $"{Discriminator:0000}" : "0000";
// This effectively represents the user's true identity.
// In the old system, this is formed from the username and discriminator.
// In the new system, the username is already the user's unique identifier.
public string FullName => Discriminator is not null
? $"{Name}#{DiscriminatorFormatted}"
: Name;
public string FullName => Discriminator is not null ? $"{Name}#{DiscriminatorFormatted}" : Name;
}
public partial record User
@@ -36,24 +34,23 @@ public partial record User
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBooleanOrNull() ?? false;
var discriminator = json
.GetPropertyOrNull("discriminator")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(int.Parse)
var discriminator = json.GetPropertyOrNull("discriminator")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(int.Parse)
.NullIfDefault();
var name = json.GetProperty("username").GetNonNullString();
var displayName = json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var displayName =
json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var avatarIndex = discriminator % 5 ?? (int)((id.Value >> 22) % 6);
var avatarUrl =
json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h))
?? ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
return new User(id, isBot, discriminator, name, displayName, avatarUrl);
}
}
}

232
DiscordChatExporter.Core/Discord/DiscordClient.cs
View File

@@ -30,48 +30,46 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
TokenKind tokenKind,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot
? $"Bot {_token}"
: _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response
.Headers
.TryGetValue("X-RateLimit-Remaining")?
.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response
.Headers
.TryGetValue("X-RateLimit-Reset-After")?
.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
return await Http.ResponseResiliencePolicy.ExecuteAsync(
async innerCancellationToken =>
{
var delay =
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot ? $"Bot {_token}" : _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response.Headers
.TryGetValue("X-RateLimit-Remaining")
?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response.Headers
.TryGetValue("X-RateLimit-Reset-After")
?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
{
var delay =
// Adding a small buffer to the reset time reduces the chance of getting
// rate limited again, because it allows for more requests to be released.
(resetAfterDelay.Value + TimeSpan.FromSeconds(1))
@@ -79,14 +77,18 @@ public class DiscordClient
// is not actually enforced by the server. So we cap it at a reasonable value.
.Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60));
await Task.Delay(delay, innerCancellationToken);
}
await Task.Delay(delay, innerCancellationToken);
}
return response;
}, cancellationToken);
return response;
},
cancellationToken
);
}
private async ValueTask<TokenKind> GetTokenKindAsync(CancellationToken cancellationToken = default)
private async ValueTask<TokenKind> GetTokenKindAsync(
CancellationToken cancellationToken = default
)
{
// Try authenticating as a user
using var userResponse = await GetResponseAsync(
@@ -113,7 +115,8 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
return await GetResponseAsync(url, tokenKind, cancellationToken);
@@ -121,7 +124,8 @@ public class DiscordClient
private async ValueTask<JsonElement> GetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
@@ -129,26 +133,30 @@ public class DiscordClient
{
throw response.StatusCode switch
{
HttpStatusCode.Unauthorized => throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Unauthorized
=> throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Forbidden => throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.Forbidden
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.NotFound => throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
HttpStatusCode.NotFound
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
_ => throw new DiscordChatExporterException(
$"""
_
=> throw new DiscordChatExporterException(
$"""
Request to '{url}' failed: {response.StatusCode.ToString().ToSpaceSeparatedWords().ToLowerInvariant()}.
Response content: {await response.Content.ReadAsStringAsync(cancellationToken)}
""",
true
)
true
)
};
}
@@ -157,7 +165,8 @@ public class DiscordClient
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
return response.IsSuccessStatusCode
@@ -167,14 +176,16 @@ public class DiscordClient
public async ValueTask<User?> TryGetUserAsync(
Snowflake userId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"users/{userId}", cancellationToken);
return response?.Pipe(User.Parse);
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
yield return Guild.DirectMessages;
@@ -206,7 +217,8 @@ public class DiscordClient
public async ValueTask<Guild> GetGuildAsync(
Snowflake guildId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
@@ -217,7 +229,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
{
@@ -227,7 +240,10 @@ public class DiscordClient
}
else
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/channels",
cancellationToken
);
var channelsJson = response
.EnumerateArray()
@@ -247,9 +263,9 @@ public class DiscordClient
foreach (var channelJson in channelsJson)
{
var parent = channelJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(channelJson, parent, position);
@@ -261,7 +277,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildThreadsAsync(
Snowflake guildId,
bool includeArchived = false,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -289,7 +306,9 @@ public class DiscordClient
if (response is null)
break;
foreach (var threadJson in response.Value.GetProperty("threads").EnumerateArray())
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
currentOffset++;
@@ -319,7 +338,9 @@ public class DiscordClient
if (response is null)
break;
foreach (var threadJson in response.Value.GetProperty("threads").EnumerateArray())
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
currentOffset++;
@@ -338,13 +359,16 @@ public class DiscordClient
{
var parentsById = channels.ToDictionary(c => c.Id);
var response = await GetJsonResponseAsync($"guilds/{guildId}/threads/active", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/threads/active",
cancellationToken
);
foreach (var threadJson in response.GetProperty("threads").EnumerateArray())
{
var parent = threadJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(threadJson, parent);
@@ -384,7 +408,8 @@ public class DiscordClient
public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -397,18 +422,23 @@ public class DiscordClient
public async ValueTask<Member?> TryGetGuildMemberAsync(
Snowflake guildId,
Snowflake memberId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return null;
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{memberId}", cancellationToken);
var response = await TryGetJsonResponseAsync(
$"guilds/{guildId}/members/{memberId}",
cancellationToken
);
return response?.Pipe(j => Member.Parse(j, guildId));
}
public async ValueTask<Invite?> TryGetInviteAsync(
string code,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"invites/{code}", cancellationToken);
return response?.Pipe(Invite.Parse);
@@ -416,14 +446,15 @@ public class DiscordClient
public async ValueTask<Channel> GetChannelAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
var parentId = response
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
try
{
@@ -445,7 +476,8 @@ public class DiscordClient
private async ValueTask<Message?> TryGetLastMessageAsync(
Snowflake channelId,
Snowflake? before = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
@@ -462,7 +494,8 @@ public class DiscordClient
Snowflake? after = null,
Snowflake? before = null,
IProgress<Percentage>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
// Get the last message in the specified range, so we can later calculate the
// progress based on the difference between message timestamps.
@@ -511,13 +544,15 @@ public class DiscordClient
var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
progress.Report(Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
));
progress.Report(
Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
)
);
}
yield return message;
@@ -530,7 +565,8 @@ public class DiscordClient
Snowflake channelId,
Snowflake messageId,
Emoji emoji,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
var reactionName = emoji.Id is not null
// Custom emoji
@@ -542,7 +578,9 @@ public class DiscordClient
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages/{messageId}/reactions/{Uri.EscapeDataString(reactionName)}")
.SetPath(
$"channels/{channelId}/messages/{messageId}/reactions/{Uri.EscapeDataString(reactionName)}"
)
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", currentAfter.ToString())
.Build();
@@ -565,4 +603,4 @@ public class DiscordClient
yield break;
}
}
}
}

14
DiscordChatExporter.Core/Discord/Snowflake.cs
View File

@@ -6,9 +6,10 @@ namespace DiscordChatExporter.Core.Discord;
public readonly partial record struct Snowflake(ulong Value)
{
public DateTimeOffset ToDate() => DateTimeOffset.FromUnixTimeMilliseconds(
(long)((Value >> 22) + 1420070400000UL)
).ToLocalTime();
public DateTimeOffset ToDate() =>
DateTimeOffset
.FromUnixTimeMilliseconds((long)((Value >> 22) + 1420070400000UL))
.ToLocalTime();
[ExcludeFromCodeCoverage]
public override string ToString() => Value.ToString(CultureInfo.InvariantCulture);
@@ -18,9 +19,8 @@ public partial record struct Snowflake
{
public static Snowflake Zero { get; } = new(0);
public static Snowflake FromDate(DateTimeOffset instant) => new(
((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22
);
public static Snowflake FromDate(DateTimeOffset instant) =>
new(((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22);
public static Snowflake? TryParse(string? str, IFormatProvider? formatProvider = null)
{
@@ -59,4 +59,4 @@ public partial record struct Snowflake : IComparable<Snowflake>, IComparable
public static bool operator >(Snowflake left, Snowflake right) => left.CompareTo(right) > 0;
public static bool operator <(Snowflake left, Snowflake right) => left.CompareTo(right) < 0;
}
}

2
DiscordChatExporter.Core/Discord/TokenKind.cs
View File

@@ -4,4 +4,4 @@ public enum TokenKind
{
User,
Bot
}
}

1
DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
View File

@@ -2,6 +2,7 @@
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" Version="6.2.1" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="Polly" Version="7.2.4" />

2
DiscordChatExporter.Core/Exceptions/DiscordChatExporterException.cs
View File

@@ -11,4 +11,4 @@ public class DiscordChatExporterException : Exception
{
IsFatal = isFatal;
}
}
}

24
DiscordChatExporter.Core/Exporting/ChannelExporter.cs
View File

@@ -16,14 +16,13 @@ public class ChannelExporter
public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<Percentage>? progress = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
// Check if the channel is empty
if (request.Channel.LastMessageId is null)
{
throw new DiscordChatExporterException(
"Channel does not contain any messages."
);
throw new DiscordChatExporterException("Channel does not contain any messages.");
}
// Check if the 'after' boundary is valid
@@ -40,12 +39,15 @@ public class ChannelExporter
// Export messages
await using var messageExporter = new MessageExporter(context);
await foreach (var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken))
await foreach (
var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken
)
)
{
// Resolve members for referenced users
foreach (var user in message.GetReferencedUsers())
@@ -64,4 +66,4 @@ public class ChannelExporter
);
}
}
}
}

29
DiscordChatExporter.Core/Exporting/CsvMessageWriter.cs
View File

@@ -20,17 +20,20 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
) => await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -48,7 +51,8 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -70,7 +74,8 @@ internal partial class CsvMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -89,15 +94,13 @@ internal partial class CsvMessageWriter : MessageWriter
// Message content
if (message.Kind.IsSystemNotification())
{
await _writer.WriteAsync(CsvEncode(
message.GetFallbackContent()
));
await _writer.WriteAsync(CsvEncode(message.GetFallbackContent()));
}
else
{
await _writer.WriteAsync(CsvEncode(
await FormatMarkdownAsync(message.Content, cancellationToken)
));
await _writer.WriteAsync(
CsvEncode(await FormatMarkdownAsync(message.Content, cancellationToken))
);
}
await _writer.WriteAsync(',');
@@ -127,4 +130,4 @@ internal partial class CsvMessageWriter
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
}
}

51
DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs
View File

@@ -15,11 +15,12 @@ namespace DiscordChatExporter.Core.Exporting;
internal partial class ExportAssetDownloader
{
private static readonly AsyncKeyedLocker<string> Locker = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly AsyncKeyedLocker<string> Locker =
new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private readonly string _workingDirPath;
private readonly bool _reuse;
@@ -33,7 +34,10 @@ internal partial class ExportAssetDownloader
_reuse = reuse;
}
public async ValueTask<string> DownloadAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> DownloadAsync(
string url,
CancellationToken cancellationToken = default
)
{
var fileName = GetFileNameFromUrl(url);
var filePath = Path.Combine(_workingDirPath, fileName);
@@ -59,11 +63,19 @@ internal partial class ExportAssetDownloader
// Try to set the file date according to the last-modified header
try
{
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var instant)
? instant
: (DateTimeOffset?)null
);
var lastModified = response.Content.Headers
.TryGetValue("Last-Modified")
?.Pipe(
s =>
DateTimeOffset.TryParse(
s,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var instant
)
? instant
: (DateTimeOffset?)null
);
if (lastModified is not null)
{
@@ -86,11 +98,12 @@ internal partial class ExportAssetDownloader
internal partial class ExportAssetDownloader
{
private static string GetUrlHash(string url) => SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetUrlHash(string url) =>
SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetFileNameFromUrl(string url)
{
@@ -115,6 +128,8 @@ internal partial class ExportAssetDownloader
fileExtension = "";
}
return PathEx.EscapeFileName(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
return PathEx.EscapeFileName(
fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension
);
}
}
}

74
DiscordChatExporter.Core/Exporting/ExportContext.cs
View File

@@ -23,8 +23,7 @@ internal class ExportContext
public ExportRequest Request { get; }
public ExportContext(DiscordClient discord,
ExportRequest request)
public ExportContext(DiscordClient discord, ExportRequest request)
{
Discord = discord;
Request = request;
@@ -35,9 +34,13 @@ internal class ExportContext
);
}
public async ValueTask PopulateChannelsAndRolesAsync(CancellationToken cancellationToken = default)
public async ValueTask PopulateChannelsAndRolesAsync(
CancellationToken cancellationToken = default
)
{
await foreach (var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken))
await foreach (
var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken)
)
_channelsById[channel.Id] = channel;
await foreach (var role in Discord.GetGuildRolesAsync(Request.Guild.Id, cancellationToken))
@@ -48,7 +51,8 @@ internal class ExportContext
private async ValueTask PopulateMemberAsync(
Snowflake id,
User? fallbackUser,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (_membersById.ContainsKey(id))
return;
@@ -70,18 +74,23 @@ internal class ExportContext
_membersById[id] = member;
}
public async ValueTask PopulateMemberAsync(Snowflake id, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(
Snowflake id,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(User user, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(user.Id, user, cancellationToken);
public async ValueTask PopulateMemberAsync(
User user,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(user.Id, user, cancellationToken);
public string FormatDate(DateTimeOffset instant) => Request.DateFormat switch
{
"unix" => instant.ToUnixTimeSeconds().ToString(),
"unixms" => instant.ToUnixTimeMilliseconds().ToString(),
var format => instant.ToLocalString(format)
};
public string FormatDate(DateTimeOffset instant) =>
Request.DateFormat switch
{
"unix" => instant.ToUnixTimeSeconds().ToString(),
"unixms" => instant.ToUnixTimeMilliseconds().ToString(),
var format => instant.ToLocalString(format)
};
public Member? TryGetMember(Snowflake id) => _membersById.GetValueOrDefault(id);
@@ -89,19 +98,20 @@ internal class ExportContext
public Role? TryGetRole(Snowflake id) => _rolesById.GetValueOrDefault(id);
public IReadOnlyList<Role> GetUserRoles(Snowflake id) => TryGetMember(id)?
.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public IReadOnlyList<Role> GetUserRoles(Snowflake id) =>
TryGetMember(id)?.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public Color? TryGetUserColor(Snowflake id) => GetUserRoles(id)
.Where(r => r.Color is not null)
.Select(r => r.Color)
.FirstOrDefault();
public Color? TryGetUserColor(Snowflake id) =>
GetUserRoles(id).Where(r => r.Color is not null).Select(r => r.Color).FirstOrDefault();
public async ValueTask<string> ResolveAssetUrlAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> ResolveAssetUrlAsync(
string url,
CancellationToken cancellationToken = default
)
{
if (!Request.ShouldDownloadAssets)
return url;
@@ -114,8 +124,14 @@ internal class ExportContext
// Prefer relative paths so that the output files can be copied around without breaking references.
// If the asset directory is outside of the export directory, use an absolute path instead.
var optimalFilePath =
relativeFilePath.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) ||
relativeFilePath.StartsWith(".." + Path.AltDirectorySeparatorChar, StringComparison.Ordinal)
relativeFilePath.StartsWith(
".." + Path.DirectorySeparatorChar,
StringComparison.Ordinal
)
|| relativeFilePath.StartsWith(
".." + Path.AltDirectorySeparatorChar,
StringComparison.Ordinal
)
? filePath
: relativeFilePath;
@@ -138,4 +154,4 @@ internal class ExportContext
return url;
}
}
}
}

40
DiscordChatExporter.Core/Exporting/ExportFormat.cs
View File

@@ -13,23 +13,25 @@ public enum ExportFormat
public static class ExportFormatExtensions
{
public static string GetFileExtension(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetFileExtension(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetDisplayName(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
public static string GetDisplayName(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}

85
DiscordChatExporter.Core/Exporting/ExportRequest.cs
View File

@@ -54,7 +54,8 @@ public partial class ExportRequest
bool shouldFormatMarkdown,
bool shouldDownloadAssets,
bool shouldReuseAssets,
string dateFormat)
string dateFormat
)
{
Guild = guild;
Channel = channel;
@@ -68,25 +69,12 @@ public partial class ExportRequest
ShouldReuseAssets = shouldReuseAssets;
DateFormat = dateFormat;
OutputFilePath = GetOutputBaseFilePath(
Guild,
Channel,
outputPath,
Format,
After,
Before
);
OutputFilePath = GetOutputBaseFilePath(Guild, Channel, outputPath, Format, After, Before);
OutputDirPath = Path.GetDirectoryName(OutputFilePath)!;
AssetsDirPath = !string.IsNullOrWhiteSpace(assetsDirPath)
? FormatPath(
assetsDirPath,
Guild,
Channel,
After,
Before
)
? FormatPath(assetsDirPath, Guild, Channel, After, Before)
: $"{OutputFilePath}_Files{Path.DirectorySeparatorChar}";
}
}
@@ -98,7 +86,8 @@ public partial class ExportRequest
Channel channel,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var buffer = new StringBuilder();
@@ -113,7 +102,9 @@ public partial class ExportRequest
// Both 'after' and 'before' are set
if (after is not null && before is not null)
{
buffer.Append($"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}");
buffer.Append(
$"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}"
);
}
// Only 'after' is set
else if (after is not null)
@@ -140,27 +131,41 @@ public partial class ExportRequest
Guild guild,
Channel channel,
Snowflake? after,
Snowflake? before)
Snowflake? before
)
{
return Regex.Replace(
path,
"%.",
m => PathEx.EscapeFileName(m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%P" => channel.Parent?.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%a" => after?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) ?? "",
"%b" => before?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) ?? "",
"%d" => DateTimeOffset.Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
"%%" => "%",
_ => m.Value
})
m =>
PathEx.EscapeFileName(
m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%P"
=> channel.Parent?.Position?.ToString(CultureInfo.InvariantCulture)
?? "0",
"%a"
=> after?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%b"
=> before?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%d"
=> DateTimeOffset.Now.ToString(
"yyyy-MM-dd",
CultureInfo.InvariantCulture
),
"%%" => "%",
_ => m.Value
}
)
);
}
@@ -170,12 +175,16 @@ public partial class ExportRequest
string outputPath,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var actualOutputPath = FormatPath(outputPath, guild, channel, after, before);
// Output is a directory
if (Directory.Exists(actualOutputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath)))
if (
Directory.Exists(actualOutputPath)
|| string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath))
)
{
var fileName = GetDefaultOutputFileName(guild, channel, format, after, before);
return Path.Combine(actualOutputPath, fileName);
@@ -184,4 +193,4 @@ public partial class ExportRequest
// Output is a file
return actualOutputPath;
}
}
}

2
DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionKind.cs
View File

@@ -4,4 +4,4 @@ internal enum BinaryExpressionKind
{
Or,
And
}
}

21
DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionMessageFilter.cs
View File

@@ -9,17 +9,22 @@ internal class BinaryExpressionMessageFilter : MessageFilter
private readonly MessageFilter _second;
private readonly BinaryExpressionKind _kind;
public BinaryExpressionMessageFilter(MessageFilter first, MessageFilter second, BinaryExpressionKind kind)
public BinaryExpressionMessageFilter(
MessageFilter first,
MessageFilter second,
BinaryExpressionKind kind
)
{
_first = first;
_second = second;
_kind = kind;
}
public override bool IsMatch(Message message) => _kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}

28
DiscordChatExporter.Core/Exporting/Filtering/ContainsMessageFilter.cs
View File

@@ -17,25 +17,21 @@ internal class ContainsMessageFilter : MessageFilter
// parentheses are not considered word characters.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909
private bool IsMatch(string? content) =>
!string.IsNullOrWhiteSpace(content) &&
Regex.IsMatch(
!string.IsNullOrWhiteSpace(content)
&& Regex.IsMatch(
content,
@"(?:\b|\s|^)" +
Regex.Escape(_text) +
@"(?:\b|\s|$)",
@"(?:\b|\s|^)" + Regex.Escape(_text) + @"(?:\b|\s|$)",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
);
public override bool IsMatch(Message message) =>
IsMatch(message.Content) ||
message.Embeds.Any(e =>
IsMatch(e.Title) ||
IsMatch(e.Author?.Name) ||
IsMatch(e.Description) ||
IsMatch(e.Footer?.Text) ||
e.Fields.Any(f =>
IsMatch(f.Name) ||
IsMatch(f.Value)
)
IsMatch(message.Content)
|| message.Embeds.Any(
e =>
IsMatch(e.Title)
|| IsMatch(e.Author?.Name)
|| IsMatch(e.Description)
|| IsMatch(e.Footer?.Text)
|| e.Fields.Any(f => IsMatch(f.Name) || IsMatch(f.Value))
);
}
}

10
DiscordChatExporter.Core/Exporting/Filtering/FromMessageFilter.cs
View File

@@ -10,8 +10,8 @@ internal class FromMessageFilter : MessageFilter
public FromMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) =>
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}

29
DiscordChatExporter.Core/Exporting/Filtering/HasMessageFilter.cs
View File

@@ -11,15 +11,20 @@ internal class HasMessageFilter : MessageFilter
public HasMessageFilter(MessageContentMatchKind kind) => _kind = kind;
public override bool IsMatch(Message message) => _kind switch
{
MessageContentMatchKind.Link => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
MessageContentMatchKind.Link
=> Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_
=> throw new InvalidOperationException(
$"Unknown message content match kind '{_kind}'."
)
};
}

16
DiscordChatExporter.Core/Exporting/Filtering/MentionsMessageFilter.cs
View File

@@ -10,10 +10,12 @@ internal class MentionsMessageFilter : MessageFilter
public MentionsMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.MentionedUsers.Any(user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
public override bool IsMatch(Message message) =>
message.MentionedUsers.Any(
user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}

2
DiscordChatExporter.Core/Exporting/Filtering/MessageContentMatchKind.cs
View File

@@ -9,4 +9,4 @@ internal enum MessageContentMatchKind
Image,
Sound,
Pin
}
}

2
DiscordChatExporter.Core/Exporting/Filtering/MessageFilter.cs
View File

@@ -14,4 +14,4 @@ public partial class MessageFilter
public static MessageFilter Null { get; } = new NullMessageFilter();
public static MessageFilter Parse(string value) => FilterGrammar.Filter.Parse(value);
}
}

2
DiscordChatExporter.Core/Exporting/Filtering/NegatedMessageFilter.cs
View File

@@ -9,4 +9,4 @@ internal class NegatedMessageFilter : MessageFilter
public NegatedMessageFilter(MessageFilter filter) => _filter = filter;
public override bool IsMatch(Message message) => !_filter.IsMatch(message);
}
}

2
DiscordChatExporter.Core/Exporting/Filtering/NullMessageFilter.cs
View File

@@ -5,4 +5,4 @@ namespace DiscordChatExporter.Core.Exporting.Filtering;
internal class NullMessageFilter : MessageFilter
{
public override bool IsMatch(Message message) => true;
}
}

160
DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterGrammar.cs
View File

@@ -6,8 +6,9 @@ namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing;
internal static class FilterGrammar
{
private static readonly TextParser<char> EscapedCharacter =
Character.EqualTo('\\').IgnoreThen(Character.AnyChar);
private static readonly TextParser<char> EscapedCharacter = Character
.EqualTo('\\')
.IgnoreThen(Character.AnyChar);
private static readonly TextParser<string> QuotedString =
from open in Character.In('"', '\'')
@@ -15,70 +16,77 @@ internal static class FilterGrammar
from close in Character.EqualTo(open)
select value;
private static readonly TextParser<string> UnquotedString =
Parse.OneOf(
private static readonly TextParser<string> UnquotedString = Parse
.OneOf(
EscapedCharacter,
// Avoid whitespace as it's treated as an implicit 'and' operator.
// Also avoid all special tokens used by other parsers.
Character.ExceptIn(' ', '(', ')', '"', '\'', '-', '~', '|', '&')
).AtLeastOnce().Text();
)
.AtLeastOnce()
.Text();
private static readonly TextParser<string> String =
Parse.OneOf(QuotedString, UnquotedString).Named("text string");
private static readonly TextParser<string> String = Parse
.OneOf(QuotedString, UnquotedString)
.Named("text string");
private static readonly TextParser<MessageFilter> ContainsFilter =
String.Select(v => (MessageFilter)new ContainsMessageFilter(v));
private static readonly TextParser<MessageFilter> ContainsFilter = String.Select(
v => (MessageFilter)new ContainsMessageFilter(v)
);
private static readonly TextParser<MessageFilter> FromFilter =
Span
.EqualToIgnoreCase("from:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> FromFilter = Span.EqualToIgnoreCase("from:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter =
Span
.EqualToIgnoreCase("mentions:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter = Span.EqualToIgnoreCase(
"mentions:"
)
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> ReactionFilter =
Span
.EqualToIgnoreCase("reaction:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new ReactionMessageFilter(v))
.Named("reaction:<value>");
private static readonly TextParser<MessageFilter> ReactionFilter = Span.EqualToIgnoreCase(
"reaction:"
)
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new ReactionMessageFilter(v))
.Named("reaction:<value>");
private static readonly TextParser<MessageFilter> HasFilter =
Span
.EqualToIgnoreCase("has:")
.Try()
.IgnoreThen(Parse.OneOf(
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file").IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound)),
private static readonly TextParser<MessageFilter> HasFilter = Span.EqualToIgnoreCase("has:")
.Try()
.IgnoreThen(
Parse.OneOf(
Span.EqualToIgnoreCase("link")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file")
.IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Sound)),
Span.EqualToIgnoreCase("pin").IgnoreThen(Parse.Return(MessageContentMatchKind.Pin))
))
.Select(k => (MessageFilter)new HasMessageFilter(k))
.Named("has:<value>");
)
)
.Select(k => (MessageFilter)new HasMessageFilter(k))
.Named("has:<value>");
// Make sure that property-based filters like 'has:link' don't prevent text like 'hello' from being parsed.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909#issuecomment-1227575455
private static readonly TextParser<MessageFilter> PrimitiveFilter =
Parse.OneOf(
FromFilter,
MentionsFilter,
ReactionFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> PrimitiveFilter = Parse.OneOf(
FromFilter,
MentionsFilter,
ReactionFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> GroupedFilter =
from open in Character.EqualTo('(')
@@ -86,36 +94,30 @@ internal static class FilterGrammar
from close in Character.EqualTo(')')
select content;
private static readonly TextParser<MessageFilter> NegatedFilter =
Character
// Dash is annoying to use from CLI due to conflicts with options, so we provide tilde as an alias
.In('-', '~')
.IgnoreThen(Parse.OneOf(GroupedFilter, PrimitiveFilter))
.Select(f => (MessageFilter)new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> NegatedFilter = Character
// Dash is annoying to use from CLI due to conflicts with options, so we provide tilde as an alias
.In('-', '~')
.IgnoreThen(Parse.OneOf(GroupedFilter, PrimitiveFilter))
.Select(f => (MessageFilter)new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> ChainedFilter =
Parse.Chain(
// Operator
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.EqualTo(' ').AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
// Operand
Parse.OneOf(
NegatedFilter,
GroupedFilter,
PrimitiveFilter
),
// Reducer
(op, left, right) => op switch
private static readonly TextParser<MessageFilter> ChainedFilter = Parse.Chain(
// Operator
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.EqualTo(' ').AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
// Operand
Parse.OneOf(NegatedFilter, GroupedFilter, PrimitiveFilter),
// Reducer
(op, left, right) =>
op switch
{
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
}
);
);
public static readonly TextParser<MessageFilter> Filter =
ChainedFilter.Token().AtEnd();
}
public static readonly TextParser<MessageFilter> Filter = ChainedFilter.Token().AtEnd();
}

14
DiscordChatExporter.Core/Exporting/Filtering/ReactionMessageFilter.cs
View File

@@ -10,9 +10,11 @@ internal class ReactionMessageFilter : MessageFilter
public ReactionMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.Reactions.Any(r =>
string.Equals(_value, r.Emoji.Id?.ToString(), StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Code, StringComparison.OrdinalIgnoreCase)
);
}
public override bool IsMatch(Message message) =>
message.Reactions.Any(
r =>
string.Equals(_value, r.Emoji.Id?.ToString(), StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, r.Emoji.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, r.Emoji.Code, StringComparison.OrdinalIgnoreCase)
);
}

144
DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs
View File

@@ -27,7 +27,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(HtmlEncode(text.Text));
return default;
@@ -35,53 +36,63 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var (openingTag, closingTag) = formatting.Kind switch
{
FormattingKind.Bold => (
// lang=html
"<strong>",
// lang=html
"</strong>"
),
FormattingKind.Bold
=> (
// lang=html
"<strong>",
// lang=html
"</strong>"
),
FormattingKind.Italic => (
// lang=html
"<em>",
// lang=html
"</em>"
),
FormattingKind.Italic
=> (
// lang=html
"<em>",
// lang=html
"</em>"
),
FormattingKind.Underline => (
// lang=html
"<u>",
// lang=html
"</u>"
),
FormattingKind.Underline
=> (
// lang=html
"<u>",
// lang=html
"</u>"
),
FormattingKind.Strikethrough => (
// lang=html
"<s>",
// lang=html
"</s>"
),
FormattingKind.Strikethrough
=> (
// lang=html
"<s>",
// lang=html
"</s>"
),
FormattingKind.Spoiler => (
// lang=html
"""<span class="chatlog__markdown-spoiler chatlog__markdown-spoiler--hidden" onclick="showSpoiler(event, this)">""",
// lang=html
"""</span>"""
),
FormattingKind.Spoiler
=> (
// lang=html
"""<span class="chatlog__markdown-spoiler chatlog__markdown-spoiler--hidden" onclick="showSpoiler(event, this)">""",
// lang=html
"""</span>"""
),
FormattingKind.Quote => (
// lang=html
"""<div class="chatlog__markdown-quote"><div class="chatlog__markdown-quote-border"></div><div class="chatlog__markdown-quote-content">""",
// lang=html
"""</div></div>"""
),
FormattingKind.Quote
=> (
// lang=html
"""<div class="chatlog__markdown-quote"><div class="chatlog__markdown-quote-border"></div><div class="chatlog__markdown-quote-content">""",
// lang=html
"""</div></div>"""
),
_ => throw new InvalidOperationException($"Unknown formatting kind '{formatting.Kind}'.")
_
=> throw new InvalidOperationException(
$"Unknown formatting kind '{formatting.Kind}'."
)
};
_buffer.Append(openingTag);
@@ -91,7 +102,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitHeadingAsync(
HeadingNode heading,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -108,7 +120,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitListAsync(
ListNode list,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -125,7 +138,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitListItemAsync(
ListItemNode listItem,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -142,7 +156,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -156,7 +171,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var highlightClass = !string.IsNullOrWhiteSpace(multiLineCodeBlock.Language)
? $"language-{multiLineCodeBlock.Language}"
@@ -174,13 +190,13 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
// Try to extract the message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(
link.Url,
@"^https?://(?:discord|discordapp)\.com/channels/.*?/(\d+)/?$"
).Groups[1].Value;
var linkedMessageId = Regex
.Match(link.Url, @"^https?://(?:discord|discordapp)\.com/channels/.*?/(\d+)/?$")
.Groups[1].Value;
_buffer.Append(
!string.IsNullOrWhiteSpace(linkedMessageId)
@@ -200,7 +216,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "chatlog__emoji--large" : "";
@@ -218,8 +235,10 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
);
}
protected override async ValueTask VisitMentionAsync(MentionNode mention,
CancellationToken cancellationToken = default)
protected override async ValueTask VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default
)
{
if (mention.Kind == MentionKind.Everyone)
{
@@ -294,7 +313,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTimestampAsync(
TimestampNode timestamp,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var formatted = timestamp.Instant is not null
? !string.IsNullOrWhiteSpace(timestamp.Format)
@@ -323,17 +343,25 @@ internal partial class HtmlMarkdownVisitor
ExportContext context,
string markdown,
bool isJumboAllowed = true,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var nodes = MarkdownParser.Parse(markdown);
var isJumbo =
isJumboAllowed &&
nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
isJumboAllowed
&& nodes.All(
n =>
n is EmojiNode
|| n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text)
);
var buffer = new StringBuilder();
await new HtmlMarkdownVisitor(context, buffer, isJumbo).VisitAsync(nodes, cancellationToken);
await new HtmlMarkdownVisitor(context, buffer, isJumbo).VisitAsync(
nodes,
cancellationToken
);
return buffer.ToString();
}
}
}

7
DiscordChatExporter.Core/Exporting/HtmlMessageExtensions.cs
View File

@@ -15,8 +15,7 @@ internal static class HtmlMessageExtensions
var embed = message.Embeds[0];
return
string.Equals(message.Content.Trim(), embed.Url, StringComparison.OrdinalIgnoreCase) &&
embed.Kind is EmbedKind.Image or EmbedKind.Gifv;
return string.Equals(message.Content.Trim(), embed.Url, StringComparison.OrdinalIgnoreCase)
&& embed.Kind is EmbedKind.Image or EmbedKind.Gifv;
}
}
}

23
DiscordChatExporter.Core/Exporting/HtmlMessageWriter.cs
View File

@@ -58,7 +58,13 @@ internal class HtmlMessageWriter : MessageWriter
// If the author changed their name after the last message, their new messages
// cannot join the existing group.
if (!string.Equals(message.Author.FullName, lastMessage.Author.FullName, StringComparison.Ordinal))
if (
!string.Equals(
message.Author.FullName,
lastMessage.Author.FullName,
StringComparison.Ordinal
)
)
return false;
}
@@ -69,7 +75,8 @@ internal class HtmlMessageWriter : MessageWriter
private string Minify(string html) => _minifier.Minify(html, false).MinifiedContent;
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(
Minify(
@@ -84,7 +91,8 @@ internal class HtmlMessageWriter : MessageWriter
private async ValueTask WriteMessageGroupAsync(
IReadOnlyList<Message> messages,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(
Minify(
@@ -99,7 +107,8 @@ internal class HtmlMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -118,7 +127,9 @@ internal class HtmlMessageWriter : MessageWriter
}
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePostambleAsync(
CancellationToken cancellationToken = default
)
{
// Flush current message group
if (_messageGroup.Any())
@@ -140,4 +151,4 @@ internal class HtmlMessageWriter : MessageWriter
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

137
DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs
View File

@@ -18,34 +18,39 @@ internal class JsonMessageWriter : MessageWriter
public JsonMessageWriter(Stream stream, ExportContext context)
: base(stream, context)
{
_writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/450
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
});
_writer = new Utf8JsonWriter(
stream,
new JsonWriterOptions
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/450
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
}
);
}
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
private async ValueTask WriteUserAsync(
User user,
CancellationToken cancellationToken = default)
private async ValueTask WriteUserAsync(User user, CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteString("color", Context.TryGetUserColor(user.Id)?.ToHex());
_writer.WriteBoolean("isBot", user.IsBot);
@@ -66,7 +71,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteRolesAsync(
IReadOnlyList<Role> roles,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartArray();
@@ -88,7 +94,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedAuthorAsync(
EmbedAuthor embedAuthor,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -99,7 +106,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl,
cancellationToken
)
);
}
@@ -109,7 +119,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedImageAsync(
EmbedImage embedImage,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -117,7 +128,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"url",
await Context.ResolveAssetUrlAsync(embedImage.ProxyUrl ?? embedImage.Url, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedImage.ProxyUrl ?? embedImage.Url,
cancellationToken
)
);
}
@@ -130,7 +144,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedFooterAsync(
EmbedFooter embedFooter,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -140,7 +155,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedFooter.IconProxyUrl ?? embedFooter.IconUrl,
cancellationToken
)
);
}
@@ -150,12 +168,16 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedFieldAsync(
EmbedField embedField,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("name", await FormatMarkdownAsync(embedField.Name, cancellationToken));
_writer.WriteString("value", await FormatMarkdownAsync(embedField.Value, cancellationToken));
_writer.WriteString(
"value",
await FormatMarkdownAsync(embedField.Value, cancellationToken)
);
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
@@ -164,14 +186,21 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedAsync(
Embed embed,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("title", await FormatMarkdownAsync(embed.Title ?? "", cancellationToken));
_writer.WriteString(
"title",
await FormatMarkdownAsync(embed.Title ?? "", cancellationToken)
);
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", await FormatMarkdownAsync(embed.Description ?? "", cancellationToken));
_writer.WriteString(
"description",
await FormatMarkdownAsync(embed.Description ?? "", cancellationToken)
);
if (embed.Color is not null)
_writer.WriteString("color", embed.Color.Value.ToHex());
@@ -220,7 +249,9 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
)
{
// Root object (start)
_writer.WriteStartObject();
@@ -250,7 +281,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(Context.Request.Channel.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
Context.Request.Channel.IconUrl,
cancellationToken
)
);
}
@@ -272,7 +306,8 @@ internal class JsonMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -293,7 +328,10 @@ internal class JsonMessageWriter : MessageWriter
}
else
{
_writer.WriteString("content", await FormatMarkdownAsync(message.Content, cancellationToken));
_writer.WriteString(
"content",
await FormatMarkdownAsync(message.Content, cancellationToken)
);
}
// Author
@@ -308,7 +346,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id.ToString());
_writer.WriteString("url", await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken));
_writer.WriteString(
"url",
await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken)
);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
@@ -335,7 +376,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("id", sticker.Id.ToString());
_writer.WriteString("name", sticker.Name);
_writer.WriteString("format", sticker.Format.ToString());
_writer.WriteString("sourceUrl", await Context.ResolveAssetUrlAsync(sticker.SourceUrl, cancellationToken));
_writer.WriteString(
"sourceUrl",
await Context.ResolveAssetUrlAsync(sticker.SourceUrl, cancellationToken)
);
_writer.WriteEndObject();
}
@@ -355,17 +399,23 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteString("code", reaction.Emoji.Code);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken));
_writer.WriteString(
"imageUrl",
await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken)
);
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteStartArray("users");
await foreach (var user in Context.Discord.GetMessageReactionsAsync(
Context.Request.Channel.Id,
message.Id,
reaction.Emoji,
cancellationToken))
await foreach (
var user in Context.Discord.GetMessageReactionsAsync(
Context.Request.Channel.Id,
message.Id,
reaction.Emoji,
cancellationToken
)
)
{
_writer.WriteStartObject();
@@ -374,7 +424,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteBoolean("isBot", user.IsBot);
_writer.WriteString(
@@ -431,7 +484,9 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePostambleAsync(
CancellationToken cancellationToken = default
)
{
// Message array (end)
_writer.WriteEndArray();
@@ -448,4 +503,4 @@ internal class JsonMessageWriter : MessageWriter
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}

36
DiscordChatExporter.Core/Exporting/MessageExporter.cs
View File

@@ -37,11 +37,18 @@ internal partial class MessageExporter : IAsyncDisposable
}
}
private async ValueTask<MessageWriter> GetWriterAsync(CancellationToken cancellationToken = default)
private async ValueTask<MessageWriter> GetWriterAsync(
CancellationToken cancellationToken = default
)
{
// Ensure that the partition limit has not been reached
if (_writer is not null &&
_context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten))
if (
_writer is not null
&& _context.Request.PartitionLimit.IsReached(
_writer.MessagesWritten,
_writer.BytesWritten
)
)
{
await ResetWriterAsync(cancellationToken);
_partitionIndex++;
@@ -60,7 +67,10 @@ internal partial class MessageExporter : IAsyncDisposable
return _writer = writer;
}
public async ValueTask ExportMessageAsync(Message message, CancellationToken cancellationToken = default)
public async ValueTask ExportMessageAsync(
Message message,
CancellationToken cancellationToken = default
)
{
var writer = await GetWriterAsync(cancellationToken);
await writer.WriteMessageAsync(message, cancellationToken);
@@ -84,22 +94,26 @@ internal partial class MessageExporter
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
var dirPath = Path.GetDirectoryName(baseFilePath);
return !string.IsNullOrWhiteSpace(dirPath)
? Path.Combine(dirPath, fileName)
: fileName;
return !string.IsNullOrWhiteSpace(dirPath) ? Path.Combine(dirPath, fileName) : fileName;
}
private static MessageWriter CreateMessageWriter(
string filePath,
ExportFormat format,
ExportContext context) =>
ExportContext context
) =>
format switch
{
ExportFormat.PlainText => new PlainTextMessageWriter(File.Create(filePath), context),
ExportFormat.Csv => new CsvMessageWriter(File.Create(filePath), context),
ExportFormat.HtmlDark => new HtmlMessageWriter(File.Create(filePath), context, "Dark"),
ExportFormat.HtmlLight => new HtmlMessageWriter(File.Create(filePath), context, "Light"),
ExportFormat.HtmlLight
=> new HtmlMessageWriter(File.Create(filePath), context, "Light"),
ExportFormat.Json => new JsonMessageWriter(File.Create(filePath), context),
_ => throw new ArgumentOutOfRangeException(nameof(format), $"Unknown export format '{format}'.")
_
=> throw new ArgumentOutOfRangeException(
nameof(format),
$"Unknown export format '{format}'."
)
};
}
}

13
DiscordChatExporter.Core/Exporting/MessageWriter.cs
View File

@@ -22,15 +22,20 @@ internal abstract class MessageWriter : IAsyncDisposable
Context = context;
}
public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
default;
public virtual ValueTask WriteMessageAsync(Message message, CancellationToken cancellationToken = default)
public virtual ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default
)
{
MessagesWritten++;
return default;
}
public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) =>
default;
public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync();
}
}

Some files were not shown because too many files have changed in this diff Show More