Use CSharpier

This commit is contained in:
Tyrrrz
2023-08-22 21:17:19 +03:00
parent c410e745b1
commit 20f58963a6
174 changed files with 11084 additions and 10670 deletions
@@ -15,30 +15,31 @@ public partial record Attachment(
string? Description,
int? Width,
int? Height,
FileSize FileSize) : IHasId
FileSize FileSize
) : IHasId
{
public string FileExtension => Path.GetExtension(FileName);
public bool IsImage =>
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
public bool IsVideo =>
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
public bool IsAudio =>
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
public bool IsSpoiler => FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
}
@@ -57,4 +58,4 @@ public partial record Attachment
return new Attachment(id, url, fileName, description, width, height, fileSize);
}
}
}
@@ -17,24 +17,27 @@ public partial record Channel(
string? IconUrl,
string? Topic,
bool IsArchived,
Snowflake? LastMessageId) : IHasId
Snowflake? LastMessageId
) : IHasId
{
// Used for visual backwards-compatibility with old exports, where
// channels without a parent (i.e. mostly DM channels) or channels
// with an inaccessible parent (i.e. inside private categories) had
// a fallback category created for them.
public string Category => Parent?.Name ?? Kind switch
{
ChannelKind.GuildCategory => "Category",
ChannelKind.GuildTextChat => "Text",
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildPrivateThread => "Private Thread",
ChannelKind.GuildPublicThread => "Public Thread",
ChannelKind.GuildNews => "News",
ChannelKind.GuildNewsThread => "News Thread",
_ => "Default"
};
public string Category =>
Parent?.Name
?? Kind switch
{
ChannelKind.GuildCategory => "Category",
ChannelKind.GuildTextChat => "Text",
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildPrivateThread => "Private Thread",
ChannelKind.GuildPublicThread => "Public Thread",
ChannelKind.GuildNews => "News",
ChannelKind.GuildNewsThread => "News Thread",
_ => "Default"
};
// Only needed for WPF data binding. Don't use anywhere else.
public bool IsVoice => Kind.IsVoice();
@@ -48,44 +51,41 @@ public partial record Channel
var kind = (ChannelKind)json.GetProperty("type").GetInt32();
var guildId =
json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse) ??
Guild.DirectMessages.Id;
json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse) ?? Guild.DirectMessages.Id;
var name =
// Guild channel
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ??
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull()
??
// DM channel
json.GetPropertyOrNull("recipients")?
.EnumerateArrayOrNull()?
.Select(User.Parse)
json.GetPropertyOrNull("recipients")
?.EnumerateArrayOrNull()
?.Select(User.Parse)
.Select(u => u.DisplayName)
.Pipe(s => string.Join(", ", s)) ??
.Pipe(s => string.Join(", ", s))
??
// Fallback
id.ToString();
var position =
positionHint ??
json.GetPropertyOrNull("position")?.GetInt32OrNull();
var position = positionHint ?? json.GetPropertyOrNull("position")?.GetInt32OrNull();
// Icons can only be set for group DM channels
var iconUrl = json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var iconUrl = json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var topic = json.GetPropertyOrNull("topic")?.GetStringOrNull();
var isArchived = json
.GetPropertyOrNull("thread_metadata")?
.GetPropertyOrNull("archived")?
.GetBooleanOrNull() ?? false;
var isArchived =
json.GetPropertyOrNull("thread_metadata")
?.GetPropertyOrNull("archived")
?.GetBooleanOrNull() ?? false;
var lastMessageId = json
.GetPropertyOrNull("last_message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var lastMessageId = json.GetPropertyOrNull("last_message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new Channel(
id,
@@ -100,4 +100,4 @@ public partial record Channel
lastMessageId
);
}
}
}
@@ -22,12 +22,14 @@ public static class ChannelKindExtensions
public static bool IsDirect(this ChannelKind kind) =>
kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
public static bool IsGuild(this ChannelKind kind) =>
!kind.IsDirect();
public static bool IsGuild(this ChannelKind kind) => !kind.IsDirect();
public static bool IsVoice(this ChannelKind kind) =>
kind is ChannelKind.GuildVoiceChat or ChannelKind.GuildStageVoice;
public static bool IsThread(this ChannelKind kind) =>
kind is ChannelKind.GuildNewsThread or ChannelKind.GuildPublicThread or ChannelKind.GuildPrivateThread;
}
kind
is ChannelKind.GuildNewsThread
or ChannelKind.GuildPublicThread
or ChannelKind.GuildPrivateThread;
}
@@ -41,10 +41,13 @@ public readonly partial record struct FileSize(long TotalBytes)
[ExcludeFromCodeCoverage]
public override string ToString() =>
string.Create(CultureInfo.InvariantCulture, $"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}");
string.Create(
CultureInfo.InvariantCulture,
$"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}"
);
}
public partial record struct FileSize
{
public static FileSize FromBytes(long bytes) => new(bytes);
}
}
@@ -3,4 +3,4 @@
public interface IHasId
{
Snowflake Id { get; }
}
}
@@ -19,10 +19,7 @@ public static class ImageCdn
? runes
: runes.Where(r => r.Value != 0xfe0f);
var twemojiId = string.Join(
"-",
filteredRunes.Select(r => r.Value.ToString("x"))
);
var twemojiId = string.Join("-", filteredRunes.Select(r => r.Value.ToString("x")));
return $"https://cdn.jsdelivr.net/gh/twitter/twemoji@latest/assets/svg/{twemojiId}.svg";
}
@@ -50,11 +47,16 @@ public static class ImageCdn
public static string GetFallbackUserAvatarUrl(int index = 0) =>
$"https://cdn.discordapp.com/embed/avatars/{index}.png";
public static string GetMemberAvatarUrl(Snowflake guildId, Snowflake userId, string avatarHash, int size = 512) =>
public static string GetMemberAvatarUrl(
Snowflake guildId,
Snowflake userId,
string avatarHash,
int size = 512
) =>
avatarHash.StartsWith("a_", StringComparison.Ordinal)
? $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.gif?size={size}"
: $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.png?size={size}";
public static string GetStickerUrl(Snowflake stickerId, string format = "png") =>
$"https://cdn.discordapp.com/stickers/{stickerId}.{format}";
}
}
@@ -21,7 +21,8 @@ public partial record Embed(
EmbedImage? Thumbnail,
IReadOnlyList<EmbedImage> Images,
EmbedVideo? Video,
EmbedFooter? Footer)
EmbedFooter? Footer
)
{
// Embeds can only have one image according to the API model,
// but the client can render multiple images in some cases.
@@ -41,24 +42,25 @@ public partial record Embed
var title = json.GetPropertyOrNull("title")?.GetStringOrNull();
var kind =
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>() ??
EmbedKind.Rich;
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>()
?? EmbedKind.Rich;
var url = json.GetPropertyOrNull("url")?.GetNonWhiteSpaceStringOrNull();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffsetOrNull();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var description = json.GetPropertyOrNull("description")?.GetStringOrNull();
var fields =
json.GetPropertyOrNull("fields")?.EnumerateArrayOrNull()?.Select(EmbedField.Parse).ToArray() ??
Array.Empty<EmbedField>();
json.GetPropertyOrNull("fields")
?.EnumerateArrayOrNull()
?.Select(EmbedField.Parse)
.ToArray() ?? Array.Empty<EmbedField>();
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
@@ -70,8 +72,10 @@ public partial record Embed
// with this by merging related embeds at the end of the message parsing process.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
var images =
json.GetPropertyOrNull("image")?.Pipe(EmbedImage.Parse).ToSingletonEnumerable().ToArray() ??
Array.Empty<EmbedImage>();
json.GetPropertyOrNull("image")
?.Pipe(EmbedImage.Parse)
.ToSingletonEnumerable()
.ToArray() ?? Array.Empty<EmbedImage>();
var video = json.GetPropertyOrNull("video")?.Pipe(EmbedVideo.Parse);
@@ -92,4 +96,4 @@ public partial record Embed
footer
);
}
}
}
@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure
public record EmbedAuthor(
string? Name,
string? Url,
string? IconUrl,
string? IconProxyUrl)
public record EmbedAuthor(string? Name, string? Url, string? IconUrl, string? IconProxyUrl)
{
public static EmbedAuthor Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedAuthor(
return new EmbedAuthor(name, url, iconUrl, iconProxyUrl);
}
}
}
@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure
public record EmbedField(
string Name,
string Value,
bool IsInline)
public record EmbedField(string Name, string Value, bool IsInline)
{
public static EmbedField Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedField(
return new EmbedField(name, value, isInline);
}
}
}
@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure
public record EmbedFooter(
string Text,
string? IconUrl,
string? IconProxyUrl)
public record EmbedFooter(string Text, string? IconUrl, string? IconProxyUrl)
{
public static EmbedFooter Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedFooter(
return new EmbedFooter(text, iconUrl, iconProxyUrl);
}
}
}
@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure
public record EmbedImage(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedImage(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedImage Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedImage(
return new EmbedImage(url, proxyUrl, width, height);
}
}
}
@@ -8,4 +8,4 @@ public enum EmbedKind
Video,
Gifv,
Link
}
}
@@ -4,11 +4,7 @@ using System.Text.Json;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-video-structure
public record EmbedVideo(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedVideo(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedVideo Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedVideo(
return new EmbedVideo(url, proxyUrl, width, height);
}
}
}
@@ -12,7 +12,9 @@ public partial record SpotifyTrackEmbedProjection
private static string? TryParseTrackId(string embedUrl)
{
// https://open.spotify.com/track/1LHZMWefF9502NPfArRfvP?si=3efac6ce9be04f0a
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[1].Value;
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[
1
].Value;
if (!string.IsNullOrWhiteSpace(trackId))
return trackId;
@@ -33,4 +35,4 @@ public partial record SpotifyTrackEmbedProjection
return new SpotifyTrackEmbedProjection(trackId);
}
}
}
@@ -21,4 +21,4 @@ public partial record YouTubeVideoEmbedProjection
return new YouTubeVideoEmbedProjection(videoId);
}
}
}
+10 -13
View File
@@ -14,12 +14,11 @@ public partial record Emoji(
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated,
string ImageUrl)
string ImageUrl
)
{
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => Id is not null
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
}
public partial record Emoji
@@ -39,19 +38,17 @@ public partial record Emoji
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var id = json.GetPropertyOrNull("id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
// Names may be missing on custom emoji within reactions
var name = json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var name =
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(
id,
name,
isAnimated,
imageUrl
);
return new Emoji(id, name, isAnimated, imageUrl);
}
}
}
File diff suppressed because it is too large Load Diff
+6 -11
View File
@@ -9,11 +9,8 @@ namespace DiscordChatExporter.Core.Discord.Data;
public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
{
// Direct messages are encapsulated within a special pseudo-guild for consistency
public static Guild DirectMessages { get; } = new(
Snowflake.Zero,
"Direct Messages",
ImageCdn.GetFallbackUserAvatarUrl()
);
public static Guild DirectMessages { get; } =
new(Snowflake.Zero, "Direct Messages", ImageCdn.GetFallbackUserAvatarUrl());
public static Guild Parse(JsonElement json)
{
@@ -21,12 +18,10 @@ public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
var name = json.GetProperty("name").GetNonNullString();
var iconUrl =
json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl();
json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ?? ImageCdn.GetFallbackUserAvatarUrl();
return new Guild(id, name, iconUrl);
}
}
}
@@ -15,4 +15,4 @@ public record Interaction(Snowflake Id, string Name, User User)
return new Interaction(id, name, user);
}
}
}
@@ -6,10 +6,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/invite#invite-object
public record Invite(
string Code,
Guild Guild,
Channel? Channel)
public record Invite(string Code, Guild Guild, Channel? Channel)
{
public static string? TryGetCodeFromUrl(string url) =>
Regex.Match(url, @"^https?://discord\.gg/(\w+)/?$").Groups[1].Value.NullIfWhiteSpace();
@@ -22,4 +19,4 @@ public record Invite(
return new Invite(code, guild, channel);
}
}
}
+13 -18
View File
@@ -13,7 +13,8 @@ public partial record Member(
User User,
string? DisplayName,
string? AvatarUrl,
IReadOnlyList<Snowflake> RoleIds) : IHasId
IReadOnlyList<Snowflake> RoleIds
) : IHasId
{
public Snowflake Id => User.Id;
}
@@ -28,25 +29,19 @@ public partial record Member
var user = json.GetProperty("user").Pipe(User.Parse);
var displayName = json.GetPropertyOrNull("nick")?.GetNonWhiteSpaceStringOrNull();
var roleIds = json
.GetPropertyOrNull("roles")?
.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var roleIds =
json.GetPropertyOrNull("roles")
?.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var avatarUrl = guildId is not null
? json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
? json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
: null;
return new Member(
user,
displayName,
avatarUrl,
roleIds
);
return new Member(user, displayName, avatarUrl, roleIds);
}
}
}
@@ -27,7 +27,8 @@ public partial record Message(
IReadOnlyList<User> MentionedUsers,
MessageReference? Reference,
Message? ReferencedMessage,
Interaction? Interaction) : IHasId
Interaction? Interaction
) : IHasId
{
public bool IsReplyLike => Kind == MessageKind.Reply || Interaction is not null;
@@ -70,22 +71,26 @@ public partial record Message
// Find embeds with the same URL that only contain a single image and nothing else
var trailingEmbeds = embeds
.Skip(i + 1)
.TakeWhile(e =>
e.Url == embed.Url &&
e.Timestamp is null &&
e.Author is null &&
e.Color is null &&
string.IsNullOrWhiteSpace(e.Description) &&
!e.Fields.Any() &&
e.Images.Count == 1 &&
e.Footer is null
.TakeWhile(
e =>
e.Url == embed.Url
&& e.Timestamp is null
&& e.Author is null
&& e.Color is null
&& string.IsNullOrWhiteSpace(e.Description)
&& !e.Fields.Any()
&& e.Images.Count == 1
&& e.Footer is null
)
.ToArray();
if (trailingEmbeds.Any())
{
// Concatenate all images into one embed
var images = embed.Images.Concat(trailingEmbeds.SelectMany(e => e.Images)).ToArray();
var images = embed.Images
.Concat(trailingEmbeds.SelectMany(e => e.Images))
.ToArray();
normalizedEmbeds.Add(embed with { Images = images });
i += trailingEmbeds.Length;
@@ -108,42 +113,49 @@ public partial record Message
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var kind = (MessageKind)json.GetProperty("type").GetInt32();
var flags = (MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var flags =
(MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var author = json.GetProperty("author").Pipe(User.Parse);
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json
.GetPropertyOrNull("call")?
.GetPropertyOrNull("ended_timestamp")?
.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json.GetPropertyOrNull("call")
?.GetPropertyOrNull("ended_timestamp")
?.GetDateTimeOffsetOrNull();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBooleanOrNull() ?? false;
var content = json.GetPropertyOrNull("content")?.GetStringOrNull() ?? "";
var attachments =
json.GetPropertyOrNull("attachments")?.EnumerateArrayOrNull()?.Select(Attachment.Parse).ToArray() ??
Array.Empty<Attachment>();
json.GetPropertyOrNull("attachments")
?.EnumerateArrayOrNull()
?.Select(Attachment.Parse)
.ToArray() ?? Array.Empty<Attachment>();
var embeds = NormalizeEmbeds(
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray() ??
Array.Empty<Embed>()
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray()
?? Array.Empty<Embed>()
);
var stickers =
json.GetPropertyOrNull("sticker_items")?.EnumerateArrayOrNull()?.Select(Sticker.Parse).ToArray() ??
Array.Empty<Sticker>();
json.GetPropertyOrNull("sticker_items")
?.EnumerateArrayOrNull()
?.Select(Sticker.Parse)
.ToArray() ?? Array.Empty<Sticker>();
var reactions =
json.GetPropertyOrNull("reactions")?.EnumerateArrayOrNull()?.Select(Reaction.Parse).ToArray() ??
Array.Empty<Reaction>();
json.GetPropertyOrNull("reactions")
?.EnumerateArrayOrNull()
?.Select(Reaction.Parse)
.ToArray() ?? Array.Empty<Reaction>();
var mentionedUsers =
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray() ??
Array.Empty<User>();
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray()
?? Array.Empty<User>();
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var messageReference = json.GetPropertyOrNull("message_reference")
?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var interaction = json.GetPropertyOrNull("interaction")?.Pipe(Interaction.Parse);
@@ -167,4 +179,4 @@ public partial record Message
interaction
);
}
}
}
@@ -15,4 +15,4 @@ public enum MessageFlags
HasThread = 32,
Ephemeral = 64,
Loading = 128
}
}
@@ -18,4 +18,4 @@ public enum MessageKind
public static class MessageKindExtensions
{
public static bool IsSystemNotification(this MessageKind kind) => (int)kind is >= 1 and <= 18;
}
}
@@ -9,21 +9,18 @@ public record MessageReference(Snowflake? MessageId, Snowflake? ChannelId, Snowf
{
public static MessageReference Parse(JsonElement json)
{
var messageId = json
.GetPropertyOrNull("message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var messageId = json.GetPropertyOrNull("message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var channelId = json
.GetPropertyOrNull("channel_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var channelId = json.GetPropertyOrNull("channel_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var guildId = json
.GetPropertyOrNull("guild_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new MessageReference(messageId, channelId, guildId);
}
}
}
@@ -13,4 +13,4 @@ public record Reaction(Emoji Emoji, int Count)
return new Reaction(emoji, count);
}
}
}
@@ -15,13 +15,12 @@ public record Role(Snowflake Id, string Name, int Position, Color? Color) : IHas
var name = json.GetProperty("name").GetNonNullString();
var position = json.GetProperty("position").GetInt32();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha()
.NullIf(c => c.ToRgb() <= 0);
return new Role(id, name, position, color);
}
}
}
@@ -15,14 +15,17 @@ public record Sticker(Snowflake Id, string Name, StickerFormat Format, string So
var name = json.GetProperty("name").GetNonNullString();
var format = (StickerFormat)json.GetProperty("format_type").GetInt32();
var sourceUrl = ImageCdn.GetStickerUrl(id, format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
});
var sourceUrl = ImageCdn.GetStickerUrl(
id,
format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
}
);
return new Sticker(id, name, format, sourceUrl);
}
}
}
@@ -5,4 +5,4 @@ public enum StickerFormat
Png = 1,
Apng = 2,
Lottie = 3
}
}
+15 -18
View File
@@ -15,18 +15,16 @@ public partial record User(
int? Discriminator,
string Name,
string DisplayName,
string AvatarUrl) : IHasId
string AvatarUrl
) : IHasId
{
public string DiscriminatorFormatted => Discriminator is not null
? $"{Discriminator:0000}"
: "0000";
public string DiscriminatorFormatted =>
Discriminator is not null ? $"{Discriminator:0000}" : "0000";
// This effectively represents the user's true identity.
// In the old system, this is formed from the username and discriminator.
// In the new system, the username is already the user's unique identifier.
public string FullName => Discriminator is not null
? $"{Name}#{DiscriminatorFormatted}"
: Name;
public string FullName => Discriminator is not null ? $"{Name}#{DiscriminatorFormatted}" : Name;
}
public partial record User
@@ -36,24 +34,23 @@ public partial record User
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBooleanOrNull() ?? false;
var discriminator = json
.GetPropertyOrNull("discriminator")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(int.Parse)
var discriminator = json.GetPropertyOrNull("discriminator")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(int.Parse)
.NullIfDefault();
var name = json.GetProperty("username").GetNonNullString();
var displayName = json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var displayName =
json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var avatarIndex = discriminator % 5 ?? (int)((id.Value >> 22) % 6);
var avatarUrl =
json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h))
?? ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
return new User(id, isBot, discriminator, name, displayName, avatarUrl);
}
}
}
+135 -97
View File
@@ -30,48 +30,46 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
TokenKind tokenKind,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot
? $"Bot {_token}"
: _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response
.Headers
.TryGetValue("X-RateLimit-Remaining")?
.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response
.Headers
.TryGetValue("X-RateLimit-Reset-After")?
.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
return await Http.ResponseResiliencePolicy.ExecuteAsync(
async innerCancellationToken =>
{
var delay =
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot ? $"Bot {_token}" : _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response.Headers
.TryGetValue("X-RateLimit-Remaining")
?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response.Headers
.TryGetValue("X-RateLimit-Reset-After")
?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
{
var delay =
// Adding a small buffer to the reset time reduces the chance of getting
// rate limited again, because it allows for more requests to be released.
(resetAfterDelay.Value + TimeSpan.FromSeconds(1))
@@ -79,14 +77,18 @@ public class DiscordClient
// is not actually enforced by the server. So we cap it at a reasonable value.
.Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60));
await Task.Delay(delay, innerCancellationToken);
}
await Task.Delay(delay, innerCancellationToken);
}
return response;
}, cancellationToken);
return response;
},
cancellationToken
);
}
private async ValueTask<TokenKind> GetTokenKindAsync(CancellationToken cancellationToken = default)
private async ValueTask<TokenKind> GetTokenKindAsync(
CancellationToken cancellationToken = default
)
{
// Try authenticating as a user
using var userResponse = await GetResponseAsync(
@@ -113,7 +115,8 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
return await GetResponseAsync(url, tokenKind, cancellationToken);
@@ -121,7 +124,8 @@ public class DiscordClient
private async ValueTask<JsonElement> GetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
@@ -129,26 +133,30 @@ public class DiscordClient
{
throw response.StatusCode switch
{
HttpStatusCode.Unauthorized => throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Unauthorized
=> throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Forbidden => throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.Forbidden
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.NotFound => throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
HttpStatusCode.NotFound
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
_ => throw new DiscordChatExporterException(
$"""
_
=> throw new DiscordChatExporterException(
$"""
Request to '{url}' failed: {response.StatusCode.ToString().ToSpaceSeparatedWords().ToLowerInvariant()}.
Response content: {await response.Content.ReadAsStringAsync(cancellationToken)}
""",
true
)
true
)
};
}
@@ -157,7 +165,8 @@ public class DiscordClient
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
return response.IsSuccessStatusCode
@@ -167,14 +176,16 @@ public class DiscordClient
public async ValueTask<User?> TryGetUserAsync(
Snowflake userId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"users/{userId}", cancellationToken);
return response?.Pipe(User.Parse);
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
yield return Guild.DirectMessages;
@@ -206,7 +217,8 @@ public class DiscordClient
public async ValueTask<Guild> GetGuildAsync(
Snowflake guildId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
@@ -217,7 +229,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
{
@@ -227,7 +240,10 @@ public class DiscordClient
}
else
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/channels",
cancellationToken
);
var channelsJson = response
.EnumerateArray()
@@ -247,9 +263,9 @@ public class DiscordClient
foreach (var channelJson in channelsJson)
{
var parent = channelJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(channelJson, parent, position);
@@ -261,7 +277,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildThreadsAsync(
Snowflake guildId,
bool includeArchived = false,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -289,7 +306,9 @@ public class DiscordClient
if (response is null)
break;
foreach (var threadJson in response.Value.GetProperty("threads").EnumerateArray())
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
currentOffset++;
@@ -319,7 +338,9 @@ public class DiscordClient
if (response is null)
break;
foreach (var threadJson in response.Value.GetProperty("threads").EnumerateArray())
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
currentOffset++;
@@ -338,13 +359,16 @@ public class DiscordClient
{
var parentsById = channels.ToDictionary(c => c.Id);
var response = await GetJsonResponseAsync($"guilds/{guildId}/threads/active", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/threads/active",
cancellationToken
);
foreach (var threadJson in response.GetProperty("threads").EnumerateArray())
{
var parent = threadJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(threadJson, parent);
@@ -384,7 +408,8 @@ public class DiscordClient
public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -397,18 +422,23 @@ public class DiscordClient
public async ValueTask<Member?> TryGetGuildMemberAsync(
Snowflake guildId,
Snowflake memberId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return null;
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{memberId}", cancellationToken);
var response = await TryGetJsonResponseAsync(
$"guilds/{guildId}/members/{memberId}",
cancellationToken
);
return response?.Pipe(j => Member.Parse(j, guildId));
}
public async ValueTask<Invite?> TryGetInviteAsync(
string code,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"invites/{code}", cancellationToken);
return response?.Pipe(Invite.Parse);
@@ -416,14 +446,15 @@ public class DiscordClient
public async ValueTask<Channel> GetChannelAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
var parentId = response
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
try
{
@@ -445,7 +476,8 @@ public class DiscordClient
private async ValueTask<Message?> TryGetLastMessageAsync(
Snowflake channelId,
Snowflake? before = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
@@ -462,7 +494,8 @@ public class DiscordClient
Snowflake? after = null,
Snowflake? before = null,
IProgress<Percentage>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
// Get the last message in the specified range, so we can later calculate the
// progress based on the difference between message timestamps.
@@ -511,13 +544,15 @@ public class DiscordClient
var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
progress.Report(Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
));
progress.Report(
Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
)
);
}
yield return message;
@@ -530,7 +565,8 @@ public class DiscordClient
Snowflake channelId,
Snowflake messageId,
Emoji emoji,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
var reactionName = emoji.Id is not null
// Custom emoji
@@ -542,7 +578,9 @@ public class DiscordClient
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages/{messageId}/reactions/{Uri.EscapeDataString(reactionName)}")
.SetPath(
$"channels/{channelId}/messages/{messageId}/reactions/{Uri.EscapeDataString(reactionName)}"
)
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", currentAfter.ToString())
.Build();
@@ -565,4 +603,4 @@ public class DiscordClient
yield break;
}
}
}
}
@@ -6,9 +6,10 @@ namespace DiscordChatExporter.Core.Discord;
public readonly partial record struct Snowflake(ulong Value)
{
public DateTimeOffset ToDate() => DateTimeOffset.FromUnixTimeMilliseconds(
(long)((Value >> 22) + 1420070400000UL)
).ToLocalTime();
public DateTimeOffset ToDate() =>
DateTimeOffset
.FromUnixTimeMilliseconds((long)((Value >> 22) + 1420070400000UL))
.ToLocalTime();
[ExcludeFromCodeCoverage]
public override string ToString() => Value.ToString(CultureInfo.InvariantCulture);
@@ -18,9 +19,8 @@ public partial record struct Snowflake
{
public static Snowflake Zero { get; } = new(0);
public static Snowflake FromDate(DateTimeOffset instant) => new(
((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22
);
public static Snowflake FromDate(DateTimeOffset instant) =>
new(((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22);
public static Snowflake? TryParse(string? str, IFormatProvider? formatProvider = null)
{
@@ -59,4 +59,4 @@ public partial record struct Snowflake : IComparable<Snowflake>, IComparable
public static bool operator >(Snowflake left, Snowflake right) => left.CompareTo(right) > 0;
public static bool operator <(Snowflake left, Snowflake right) => left.CompareTo(right) < 0;
}
}
@@ -4,4 +4,4 @@ public enum TokenKind
{
User,
Bot
}
}
@@ -2,6 +2,7 @@
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" Version="6.2.1" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="Polly" Version="7.2.4" />
@@ -11,4 +11,4 @@ public class DiscordChatExporterException : Exception
{
IsFatal = isFatal;
}
}
}
@@ -16,14 +16,13 @@ public class ChannelExporter
public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<Percentage>? progress = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
// Check if the channel is empty
if (request.Channel.LastMessageId is null)
{
throw new DiscordChatExporterException(
"Channel does not contain any messages."
);
throw new DiscordChatExporterException("Channel does not contain any messages.");
}
// Check if the 'after' boundary is valid
@@ -40,12 +39,15 @@ public class ChannelExporter
// Export messages
await using var messageExporter = new MessageExporter(context);
await foreach (var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken))
await foreach (
var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken
)
)
{
// Resolve members for referenced users
foreach (var user in message.GetReferencedUsers())
@@ -64,4 +66,4 @@ public class ChannelExporter
);
}
}
}
}
@@ -20,17 +20,20 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
) => await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -48,7 +51,8 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -70,7 +74,8 @@ internal partial class CsvMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -89,15 +94,13 @@ internal partial class CsvMessageWriter : MessageWriter
// Message content
if (message.Kind.IsSystemNotification())
{
await _writer.WriteAsync(CsvEncode(
message.GetFallbackContent()
));
await _writer.WriteAsync(CsvEncode(message.GetFallbackContent()));
}
else
{
await _writer.WriteAsync(CsvEncode(
await FormatMarkdownAsync(message.Content, cancellationToken)
));
await _writer.WriteAsync(
CsvEncode(await FormatMarkdownAsync(message.Content, cancellationToken))
);
}
await _writer.WriteAsync(',');
@@ -127,4 +130,4 @@ internal partial class CsvMessageWriter
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
}
}
@@ -15,11 +15,12 @@ namespace DiscordChatExporter.Core.Exporting;
internal partial class ExportAssetDownloader
{
private static readonly AsyncKeyedLocker<string> Locker = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly AsyncKeyedLocker<string> Locker =
new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private readonly string _workingDirPath;
private readonly bool _reuse;
@@ -33,7 +34,10 @@ internal partial class ExportAssetDownloader
_reuse = reuse;
}
public async ValueTask<string> DownloadAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> DownloadAsync(
string url,
CancellationToken cancellationToken = default
)
{
var fileName = GetFileNameFromUrl(url);
var filePath = Path.Combine(_workingDirPath, fileName);
@@ -59,11 +63,19 @@ internal partial class ExportAssetDownloader
// Try to set the file date according to the last-modified header
try
{
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var instant)
? instant
: (DateTimeOffset?)null
);
var lastModified = response.Content.Headers
.TryGetValue("Last-Modified")
?.Pipe(
s =>
DateTimeOffset.TryParse(
s,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var instant
)
? instant
: (DateTimeOffset?)null
);
if (lastModified is not null)
{
@@ -86,11 +98,12 @@ internal partial class ExportAssetDownloader
internal partial class ExportAssetDownloader
{
private static string GetUrlHash(string url) => SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetUrlHash(string url) =>
SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetFileNameFromUrl(string url)
{
@@ -115,6 +128,8 @@ internal partial class ExportAssetDownloader
fileExtension = "";
}
return PathEx.EscapeFileName(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
return PathEx.EscapeFileName(
fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension
);
}
}
}
@@ -23,8 +23,7 @@ internal class ExportContext
public ExportRequest Request { get; }
public ExportContext(DiscordClient discord,
ExportRequest request)
public ExportContext(DiscordClient discord, ExportRequest request)
{
Discord = discord;
Request = request;
@@ -35,9 +34,13 @@ internal class ExportContext
);
}
public async ValueTask PopulateChannelsAndRolesAsync(CancellationToken cancellationToken = default)
public async ValueTask PopulateChannelsAndRolesAsync(
CancellationToken cancellationToken = default
)
{
await foreach (var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken))
await foreach (
var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken)
)
_channelsById[channel.Id] = channel;
await foreach (var role in Discord.GetGuildRolesAsync(Request.Guild.Id, cancellationToken))
@@ -48,7 +51,8 @@ internal class ExportContext
private async ValueTask PopulateMemberAsync(
Snowflake id,
User? fallbackUser,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (_membersById.ContainsKey(id))
return;
@@ -70,18 +74,23 @@ internal class ExportContext
_membersById[id] = member;
}
public async ValueTask PopulateMemberAsync(Snowflake id, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(
Snowflake id,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(User user, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(user.Id, user, cancellationToken);
public async ValueTask PopulateMemberAsync(
User user,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(user.Id, user, cancellationToken);
public string FormatDate(DateTimeOffset instant) => Request.DateFormat switch
{
"unix" => instant.ToUnixTimeSeconds().ToString(),
"unixms" => instant.ToUnixTimeMilliseconds().ToString(),
var format => instant.ToLocalString(format)
};
public string FormatDate(DateTimeOffset instant) =>
Request.DateFormat switch
{
"unix" => instant.ToUnixTimeSeconds().ToString(),
"unixms" => instant.ToUnixTimeMilliseconds().ToString(),
var format => instant.ToLocalString(format)
};
public Member? TryGetMember(Snowflake id) => _membersById.GetValueOrDefault(id);
@@ -89,19 +98,20 @@ internal class ExportContext
public Role? TryGetRole(Snowflake id) => _rolesById.GetValueOrDefault(id);
public IReadOnlyList<Role> GetUserRoles(Snowflake id) => TryGetMember(id)?
.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public IReadOnlyList<Role> GetUserRoles(Snowflake id) =>
TryGetMember(id)?.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public Color? TryGetUserColor(Snowflake id) => GetUserRoles(id)
.Where(r => r.Color is not null)
.Select(r => r.Color)
.FirstOrDefault();
public Color? TryGetUserColor(Snowflake id) =>
GetUserRoles(id).Where(r => r.Color is not null).Select(r => r.Color).FirstOrDefault();
public async ValueTask<string> ResolveAssetUrlAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> ResolveAssetUrlAsync(
string url,
CancellationToken cancellationToken = default
)
{
if (!Request.ShouldDownloadAssets)
return url;
@@ -114,8 +124,14 @@ internal class ExportContext
// Prefer relative paths so that the output files can be copied around without breaking references.
// If the asset directory is outside of the export directory, use an absolute path instead.
var optimalFilePath =
relativeFilePath.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) ||
relativeFilePath.StartsWith(".." + Path.AltDirectorySeparatorChar, StringComparison.Ordinal)
relativeFilePath.StartsWith(
".." + Path.DirectorySeparatorChar,
StringComparison.Ordinal
)
|| relativeFilePath.StartsWith(
".." + Path.AltDirectorySeparatorChar,
StringComparison.Ordinal
)
? filePath
: relativeFilePath;
@@ -138,4 +154,4 @@ internal class ExportContext
return url;
}
}
}
}
@@ -13,23 +13,25 @@ public enum ExportFormat
public static class ExportFormatExtensions
{
public static string GetFileExtension(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetFileExtension(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetDisplayName(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
public static string GetDisplayName(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
@@ -54,7 +54,8 @@ public partial class ExportRequest
bool shouldFormatMarkdown,
bool shouldDownloadAssets,
bool shouldReuseAssets,
string dateFormat)
string dateFormat
)
{
Guild = guild;
Channel = channel;
@@ -68,25 +69,12 @@ public partial class ExportRequest
ShouldReuseAssets = shouldReuseAssets;
DateFormat = dateFormat;
OutputFilePath = GetOutputBaseFilePath(
Guild,
Channel,
outputPath,
Format,
After,
Before
);
OutputFilePath = GetOutputBaseFilePath(Guild, Channel, outputPath, Format, After, Before);
OutputDirPath = Path.GetDirectoryName(OutputFilePath)!;
AssetsDirPath = !string.IsNullOrWhiteSpace(assetsDirPath)
? FormatPath(
assetsDirPath,
Guild,
Channel,
After,
Before
)
? FormatPath(assetsDirPath, Guild, Channel, After, Before)
: $"{OutputFilePath}_Files{Path.DirectorySeparatorChar}";
}
}
@@ -98,7 +86,8 @@ public partial class ExportRequest
Channel channel,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var buffer = new StringBuilder();
@@ -113,7 +102,9 @@ public partial class ExportRequest
// Both 'after' and 'before' are set
if (after is not null && before is not null)
{
buffer.Append($"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}");
buffer.Append(
$"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}"
);
}
// Only 'after' is set
else if (after is not null)
@@ -140,27 +131,41 @@ public partial class ExportRequest
Guild guild,
Channel channel,
Snowflake? after,
Snowflake? before)
Snowflake? before
)
{
return Regex.Replace(
path,
"%.",
m => PathEx.EscapeFileName(m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%P" => channel.Parent?.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%a" => after?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) ?? "",
"%b" => before?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) ?? "",
"%d" => DateTimeOffset.Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
"%%" => "%",
_ => m.Value
})
m =>
PathEx.EscapeFileName(
m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%P"
=> channel.Parent?.Position?.ToString(CultureInfo.InvariantCulture)
?? "0",
"%a"
=> after?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%b"
=> before?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%d"
=> DateTimeOffset.Now.ToString(
"yyyy-MM-dd",
CultureInfo.InvariantCulture
),
"%%" => "%",
_ => m.Value
}
)
);
}
@@ -170,12 +175,16 @@ public partial class ExportRequest
string outputPath,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var actualOutputPath = FormatPath(outputPath, guild, channel, after, before);
// Output is a directory
if (Directory.Exists(actualOutputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath)))
if (
Directory.Exists(actualOutputPath)
|| string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath))
)
{
var fileName = GetDefaultOutputFileName(guild, channel, format, after, before);
return Path.Combine(actualOutputPath, fileName);
@@ -184,4 +193,4 @@ public partial class ExportRequest
// Output is a file
return actualOutputPath;
}
}
}
@@ -4,4 +4,4 @@ internal enum BinaryExpressionKind
{
Or,
And
}
}
@@ -9,17 +9,22 @@ internal class BinaryExpressionMessageFilter : MessageFilter
private readonly MessageFilter _second;
private readonly BinaryExpressionKind _kind;
public BinaryExpressionMessageFilter(MessageFilter first, MessageFilter second, BinaryExpressionKind kind)
public BinaryExpressionMessageFilter(
MessageFilter first,
MessageFilter second,
BinaryExpressionKind kind
)
{
_first = first;
_second = second;
_kind = kind;
}
public override bool IsMatch(Message message) => _kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
@@ -17,25 +17,21 @@ internal class ContainsMessageFilter : MessageFilter
// parentheses are not considered word characters.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909
private bool IsMatch(string? content) =>
!string.IsNullOrWhiteSpace(content) &&
Regex.IsMatch(
!string.IsNullOrWhiteSpace(content)
&& Regex.IsMatch(
content,
@"(?:\b|\s|^)" +
Regex.Escape(_text) +
@"(?:\b|\s|$)",
@"(?:\b|\s|^)" + Regex.Escape(_text) + @"(?:\b|\s|$)",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
);
public override bool IsMatch(Message message) =>
IsMatch(message.Content) ||
message.Embeds.Any(e =>
IsMatch(e.Title) ||
IsMatch(e.Author?.Name) ||
IsMatch(e.Description) ||
IsMatch(e.Footer?.Text) ||
e.Fields.Any(f =>
IsMatch(f.Name) ||
IsMatch(f.Value)
)
IsMatch(message.Content)
|| message.Embeds.Any(
e =>
IsMatch(e.Title)
|| IsMatch(e.Author?.Name)
|| IsMatch(e.Description)
|| IsMatch(e.Footer?.Text)
|| e.Fields.Any(f => IsMatch(f.Name) || IsMatch(f.Value))
);
}
}
@@ -10,8 +10,8 @@ internal class FromMessageFilter : MessageFilter
public FromMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) =>
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
@@ -11,15 +11,20 @@ internal class HasMessageFilter : MessageFilter
public HasMessageFilter(MessageContentMatchKind kind) => _kind = kind;
public override bool IsMatch(Message message) => _kind switch
{
MessageContentMatchKind.Link => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
MessageContentMatchKind.Link
=> Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_
=> throw new InvalidOperationException(
$"Unknown message content match kind '{_kind}'."
)
};
}
@@ -10,10 +10,12 @@ internal class MentionsMessageFilter : MessageFilter
public MentionsMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.MentionedUsers.Any(user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
public override bool IsMatch(Message message) =>
message.MentionedUsers.Any(
user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
@@ -9,4 +9,4 @@ internal enum MessageContentMatchKind
Image,
Sound,
Pin
}
}
@@ -14,4 +14,4 @@ public partial class MessageFilter
public static MessageFilter Null { get; } = new NullMessageFilter();
public static MessageFilter Parse(string value) => FilterGrammar.Filter.Parse(value);
}
}
@@ -9,4 +9,4 @@ internal class NegatedMessageFilter : MessageFilter
public NegatedMessageFilter(MessageFilter filter) => _filter = filter;
public override bool IsMatch(Message message) => !_filter.IsMatch(message);
}
}
@@ -5,4 +5,4 @@ namespace DiscordChatExporter.Core.Exporting.Filtering;
internal class NullMessageFilter : MessageFilter
{
public override bool IsMatch(Message message) => true;
}
}
@@ -6,8 +6,9 @@ namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing;
internal static class FilterGrammar
{
private static readonly TextParser<char> EscapedCharacter =
Character.EqualTo('\\').IgnoreThen(Character.AnyChar);
private static readonly TextParser<char> EscapedCharacter = Character
.EqualTo('\\')
.IgnoreThen(Character.AnyChar);
private static readonly TextParser<string> QuotedString =
from open in Character.In('"', '\'')
@@ -15,70 +16,77 @@ internal static class FilterGrammar
from close in Character.EqualTo(open)
select value;
private static readonly TextParser<string> UnquotedString =
Parse.OneOf(
private static readonly TextParser<string> UnquotedString = Parse
.OneOf(
EscapedCharacter,
// Avoid whitespace as it's treated as an implicit 'and' operator.
// Also avoid all special tokens used by other parsers.
Character.ExceptIn(' ', '(', ')', '"', '\'', '-', '~', '|', '&')
).AtLeastOnce().Text();
)
.AtLeastOnce()
.Text();
private static readonly TextParser<string> String =
Parse.OneOf(QuotedString, UnquotedString).Named("text string");
private static readonly TextParser<string> String = Parse
.OneOf(QuotedString, UnquotedString)
.Named("text string");
private static readonly TextParser<MessageFilter> ContainsFilter =
String.Select(v => (MessageFilter)new ContainsMessageFilter(v));
private static readonly TextParser<MessageFilter> ContainsFilter = String.Select(
v => (MessageFilter)new ContainsMessageFilter(v)
);
private static readonly TextParser<MessageFilter> FromFilter =
Span
.EqualToIgnoreCase("from:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> FromFilter = Span.EqualToIgnoreCase("from:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter =
Span
.EqualToIgnoreCase("mentions:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter = Span.EqualToIgnoreCase(
"mentions:"
)
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> ReactionFilter =
Span
.EqualToIgnoreCase("reaction:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new ReactionMessageFilter(v))
.Named("reaction:<value>");
private static readonly TextParser<MessageFilter> ReactionFilter = Span.EqualToIgnoreCase(
"reaction:"
)
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new ReactionMessageFilter(v))
.Named("reaction:<value>");
private static readonly TextParser<MessageFilter> HasFilter =
Span
.EqualToIgnoreCase("has:")
.Try()
.IgnoreThen(Parse.OneOf(
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file").IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound)),
private static readonly TextParser<MessageFilter> HasFilter = Span.EqualToIgnoreCase("has:")
.Try()
.IgnoreThen(
Parse.OneOf(
Span.EqualToIgnoreCase("link")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file")
.IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound")
.IgnoreThen(Parse.Return(MessageContentMatchKind.Sound)),
Span.EqualToIgnoreCase("pin").IgnoreThen(Parse.Return(MessageContentMatchKind.Pin))
))
.Select(k => (MessageFilter)new HasMessageFilter(k))
.Named("has:<value>");
)
)
.Select(k => (MessageFilter)new HasMessageFilter(k))
.Named("has:<value>");
// Make sure that property-based filters like 'has:link' don't prevent text like 'hello' from being parsed.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909#issuecomment-1227575455
private static readonly TextParser<MessageFilter> PrimitiveFilter =
Parse.OneOf(
FromFilter,
MentionsFilter,
ReactionFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> PrimitiveFilter = Parse.OneOf(
FromFilter,
MentionsFilter,
ReactionFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> GroupedFilter =
from open in Character.EqualTo('(')
@@ -86,36 +94,30 @@ internal static class FilterGrammar
from close in Character.EqualTo(')')
select content;
private static readonly TextParser<MessageFilter> NegatedFilter =
Character
// Dash is annoying to use from CLI due to conflicts with options, so we provide tilde as an alias
.In('-', '~')
.IgnoreThen(Parse.OneOf(GroupedFilter, PrimitiveFilter))
.Select(f => (MessageFilter)new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> NegatedFilter = Character
// Dash is annoying to use from CLI due to conflicts with options, so we provide tilde as an alias
.In('-', '~')
.IgnoreThen(Parse.OneOf(GroupedFilter, PrimitiveFilter))
.Select(f => (MessageFilter)new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> ChainedFilter =
Parse.Chain(
// Operator
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.EqualTo(' ').AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
// Operand
Parse.OneOf(
NegatedFilter,
GroupedFilter,
PrimitiveFilter
),
// Reducer
(op, left, right) => op switch
private static readonly TextParser<MessageFilter> ChainedFilter = Parse.Chain(
// Operator
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.EqualTo(' ').AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
// Operand
Parse.OneOf(NegatedFilter, GroupedFilter, PrimitiveFilter),
// Reducer
(op, left, right) =>
op switch
{
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
}
);
);
public static readonly TextParser<MessageFilter> Filter =
ChainedFilter.Token().AtEnd();
}
public static readonly TextParser<MessageFilter> Filter = ChainedFilter.Token().AtEnd();
}
@@ -10,9 +10,11 @@ internal class ReactionMessageFilter : MessageFilter
public ReactionMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.Reactions.Any(r =>
string.Equals(_value, r.Emoji.Id?.ToString(), StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Code, StringComparison.OrdinalIgnoreCase)
);
}
public override bool IsMatch(Message message) =>
message.Reactions.Any(
r =>
string.Equals(_value, r.Emoji.Id?.ToString(), StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, r.Emoji.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, r.Emoji.Code, StringComparison.OrdinalIgnoreCase)
);
}
@@ -27,7 +27,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(HtmlEncode(text.Text));
return default;
@@ -35,53 +36,63 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var (openingTag, closingTag) = formatting.Kind switch
{
FormattingKind.Bold => (
// lang=html
"<strong>",
// lang=html
"</strong>"
),
FormattingKind.Bold
=> (
// lang=html
"<strong>",
// lang=html
"</strong>"
),
FormattingKind.Italic => (
// lang=html
"<em>",
// lang=html
"</em>"
),
FormattingKind.Italic
=> (
// lang=html
"<em>",
// lang=html
"</em>"
),
FormattingKind.Underline => (
// lang=html
"<u>",
// lang=html
"</u>"
),
FormattingKind.Underline
=> (
// lang=html
"<u>",
// lang=html
"</u>"
),
FormattingKind.Strikethrough => (
// lang=html
"<s>",
// lang=html
"</s>"
),
FormattingKind.Strikethrough
=> (
// lang=html
"<s>",
// lang=html
"</s>"
),
FormattingKind.Spoiler => (
// lang=html
"""<span class="chatlog__markdown-spoiler chatlog__markdown-spoiler--hidden" onclick="showSpoiler(event, this)">""",
// lang=html
"""</span>"""
),
FormattingKind.Spoiler
=> (
// lang=html
"""<span class="chatlog__markdown-spoiler chatlog__markdown-spoiler--hidden" onclick="showSpoiler(event, this)">""",
// lang=html
"""</span>"""
),
FormattingKind.Quote => (
// lang=html
"""<div class="chatlog__markdown-quote"><div class="chatlog__markdown-quote-border"></div><div class="chatlog__markdown-quote-content">""",
// lang=html
"""</div></div>"""
),
FormattingKind.Quote
=> (
// lang=html
"""<div class="chatlog__markdown-quote"><div class="chatlog__markdown-quote-border"></div><div class="chatlog__markdown-quote-content">""",
// lang=html
"""</div></div>"""
),
_ => throw new InvalidOperationException($"Unknown formatting kind '{formatting.Kind}'.")
_
=> throw new InvalidOperationException(
$"Unknown formatting kind '{formatting.Kind}'."
)
};
_buffer.Append(openingTag);
@@ -91,7 +102,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitHeadingAsync(
HeadingNode heading,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -108,7 +120,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitListAsync(
ListNode list,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -125,7 +138,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitListItemAsync(
ListItemNode listItem,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -142,7 +156,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
// lang=html
@@ -156,7 +171,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var highlightClass = !string.IsNullOrWhiteSpace(multiLineCodeBlock.Language)
? $"language-{multiLineCodeBlock.Language}"
@@ -174,13 +190,13 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
// Try to extract the message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(
link.Url,
@"^https?://(?:discord|discordapp)\.com/channels/.*?/(\d+)/?$"
).Groups[1].Value;
var linkedMessageId = Regex
.Match(link.Url, @"^https?://(?:discord|discordapp)\.com/channels/.*?/(\d+)/?$")
.Groups[1].Value;
_buffer.Append(
!string.IsNullOrWhiteSpace(linkedMessageId)
@@ -200,7 +216,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override async ValueTask VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "chatlog__emoji--large" : "";
@@ -218,8 +235,10 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
);
}
protected override async ValueTask VisitMentionAsync(MentionNode mention,
CancellationToken cancellationToken = default)
protected override async ValueTask VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default
)
{
if (mention.Kind == MentionKind.Everyone)
{
@@ -294,7 +313,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTimestampAsync(
TimestampNode timestamp,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var formatted = timestamp.Instant is not null
? !string.IsNullOrWhiteSpace(timestamp.Format)
@@ -323,17 +343,25 @@ internal partial class HtmlMarkdownVisitor
ExportContext context,
string markdown,
bool isJumboAllowed = true,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var nodes = MarkdownParser.Parse(markdown);
var isJumbo =
isJumboAllowed &&
nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
isJumboAllowed
&& nodes.All(
n =>
n is EmojiNode
|| n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text)
);
var buffer = new StringBuilder();
await new HtmlMarkdownVisitor(context, buffer, isJumbo).VisitAsync(nodes, cancellationToken);
await new HtmlMarkdownVisitor(context, buffer, isJumbo).VisitAsync(
nodes,
cancellationToken
);
return buffer.ToString();
}
}
}
@@ -15,8 +15,7 @@ internal static class HtmlMessageExtensions
var embed = message.Embeds[0];
return
string.Equals(message.Content.Trim(), embed.Url, StringComparison.OrdinalIgnoreCase) &&
embed.Kind is EmbedKind.Image or EmbedKind.Gifv;
return string.Equals(message.Content.Trim(), embed.Url, StringComparison.OrdinalIgnoreCase)
&& embed.Kind is EmbedKind.Image or EmbedKind.Gifv;
}
}
}
@@ -58,7 +58,13 @@ internal class HtmlMessageWriter : MessageWriter
// If the author changed their name after the last message, their new messages
// cannot join the existing group.
if (!string.Equals(message.Author.FullName, lastMessage.Author.FullName, StringComparison.Ordinal))
if (
!string.Equals(
message.Author.FullName,
lastMessage.Author.FullName,
StringComparison.Ordinal
)
)
return false;
}
@@ -69,7 +75,8 @@ internal class HtmlMessageWriter : MessageWriter
private string Minify(string html) => _minifier.Minify(html, false).MinifiedContent;
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(
Minify(
@@ -84,7 +91,8 @@ internal class HtmlMessageWriter : MessageWriter
private async ValueTask WriteMessageGroupAsync(
IReadOnlyList<Message> messages,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(
Minify(
@@ -99,7 +107,8 @@ internal class HtmlMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -118,7 +127,9 @@ internal class HtmlMessageWriter : MessageWriter
}
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePostambleAsync(
CancellationToken cancellationToken = default
)
{
// Flush current message group
if (_messageGroup.Any())
@@ -140,4 +151,4 @@ internal class HtmlMessageWriter : MessageWriter
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}
@@ -18,34 +18,39 @@ internal class JsonMessageWriter : MessageWriter
public JsonMessageWriter(Stream stream, ExportContext context)
: base(stream, context)
{
_writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/450
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
});
_writer = new Utf8JsonWriter(
stream,
new JsonWriterOptions
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/450
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
}
);
}
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
private async ValueTask WriteUserAsync(
User user,
CancellationToken cancellationToken = default)
private async ValueTask WriteUserAsync(User user, CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteString("color", Context.TryGetUserColor(user.Id)?.ToHex());
_writer.WriteBoolean("isBot", user.IsBot);
@@ -66,7 +71,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteRolesAsync(
IReadOnlyList<Role> roles,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartArray();
@@ -88,7 +94,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedAuthorAsync(
EmbedAuthor embedAuthor,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -99,7 +106,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl,
cancellationToken
)
);
}
@@ -109,7 +119,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedImageAsync(
EmbedImage embedImage,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -117,7 +128,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"url",
await Context.ResolveAssetUrlAsync(embedImage.ProxyUrl ?? embedImage.Url, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedImage.ProxyUrl ?? embedImage.Url,
cancellationToken
)
);
}
@@ -130,7 +144,8 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedFooterAsync(
EmbedFooter embedFooter,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
@@ -140,7 +155,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
embedFooter.IconProxyUrl ?? embedFooter.IconUrl,
cancellationToken
)
);
}
@@ -150,12 +168,16 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedFieldAsync(
EmbedField embedField,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("name", await FormatMarkdownAsync(embedField.Name, cancellationToken));
_writer.WriteString("value", await FormatMarkdownAsync(embedField.Value, cancellationToken));
_writer.WriteString(
"value",
await FormatMarkdownAsync(embedField.Value, cancellationToken)
);
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
@@ -164,14 +186,21 @@ internal class JsonMessageWriter : MessageWriter
private async ValueTask WriteEmbedAsync(
Embed embed,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("title", await FormatMarkdownAsync(embed.Title ?? "", cancellationToken));
_writer.WriteString(
"title",
await FormatMarkdownAsync(embed.Title ?? "", cancellationToken)
);
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", await FormatMarkdownAsync(embed.Description ?? "", cancellationToken));
_writer.WriteString(
"description",
await FormatMarkdownAsync(embed.Description ?? "", cancellationToken)
);
if (embed.Color is not null)
_writer.WriteString("color", embed.Color.Value.ToHex());
@@ -220,7 +249,9 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
)
{
// Root object (start)
_writer.WriteStartObject();
@@ -250,7 +281,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteString(
"iconUrl",
await Context.ResolveAssetUrlAsync(Context.Request.Channel.IconUrl, cancellationToken)
await Context.ResolveAssetUrlAsync(
Context.Request.Channel.IconUrl,
cancellationToken
)
);
}
@@ -272,7 +306,8 @@ internal class JsonMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -293,7 +328,10 @@ internal class JsonMessageWriter : MessageWriter
}
else
{
_writer.WriteString("content", await FormatMarkdownAsync(message.Content, cancellationToken));
_writer.WriteString(
"content",
await FormatMarkdownAsync(message.Content, cancellationToken)
);
}
// Author
@@ -308,7 +346,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id.ToString());
_writer.WriteString("url", await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken));
_writer.WriteString(
"url",
await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken)
);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
@@ -335,7 +376,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("id", sticker.Id.ToString());
_writer.WriteString("name", sticker.Name);
_writer.WriteString("format", sticker.Format.ToString());
_writer.WriteString("sourceUrl", await Context.ResolveAssetUrlAsync(sticker.SourceUrl, cancellationToken));
_writer.WriteString(
"sourceUrl",
await Context.ResolveAssetUrlAsync(sticker.SourceUrl, cancellationToken)
);
_writer.WriteEndObject();
}
@@ -355,17 +399,23 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteString("code", reaction.Emoji.Code);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken));
_writer.WriteString(
"imageUrl",
await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken)
);
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteStartArray("users");
await foreach (var user in Context.Discord.GetMessageReactionsAsync(
Context.Request.Channel.Id,
message.Id,
reaction.Emoji,
cancellationToken))
await foreach (
var user in Context.Discord.GetMessageReactionsAsync(
Context.Request.Channel.Id,
message.Id,
reaction.Emoji,
cancellationToken
)
)
{
_writer.WriteStartObject();
@@ -374,7 +424,10 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteBoolean("isBot", user.IsBot);
_writer.WriteString(
@@ -431,7 +484,9 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePostambleAsync(
CancellationToken cancellationToken = default
)
{
// Message array (end)
_writer.WriteEndArray();
@@ -448,4 +503,4 @@ internal class JsonMessageWriter : MessageWriter
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}
@@ -37,11 +37,18 @@ internal partial class MessageExporter : IAsyncDisposable
}
}
private async ValueTask<MessageWriter> GetWriterAsync(CancellationToken cancellationToken = default)
private async ValueTask<MessageWriter> GetWriterAsync(
CancellationToken cancellationToken = default
)
{
// Ensure that the partition limit has not been reached
if (_writer is not null &&
_context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten))
if (
_writer is not null
&& _context.Request.PartitionLimit.IsReached(
_writer.MessagesWritten,
_writer.BytesWritten
)
)
{
await ResetWriterAsync(cancellationToken);
_partitionIndex++;
@@ -60,7 +67,10 @@ internal partial class MessageExporter : IAsyncDisposable
return _writer = writer;
}
public async ValueTask ExportMessageAsync(Message message, CancellationToken cancellationToken = default)
public async ValueTask ExportMessageAsync(
Message message,
CancellationToken cancellationToken = default
)
{
var writer = await GetWriterAsync(cancellationToken);
await writer.WriteMessageAsync(message, cancellationToken);
@@ -84,22 +94,26 @@ internal partial class MessageExporter
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
var dirPath = Path.GetDirectoryName(baseFilePath);
return !string.IsNullOrWhiteSpace(dirPath)
? Path.Combine(dirPath, fileName)
: fileName;
return !string.IsNullOrWhiteSpace(dirPath) ? Path.Combine(dirPath, fileName) : fileName;
}
private static MessageWriter CreateMessageWriter(
string filePath,
ExportFormat format,
ExportContext context) =>
ExportContext context
) =>
format switch
{
ExportFormat.PlainText => new PlainTextMessageWriter(File.Create(filePath), context),
ExportFormat.Csv => new CsvMessageWriter(File.Create(filePath), context),
ExportFormat.HtmlDark => new HtmlMessageWriter(File.Create(filePath), context, "Dark"),
ExportFormat.HtmlLight => new HtmlMessageWriter(File.Create(filePath), context, "Light"),
ExportFormat.HtmlLight
=> new HtmlMessageWriter(File.Create(filePath), context, "Light"),
ExportFormat.Json => new JsonMessageWriter(File.Create(filePath), context),
_ => throw new ArgumentOutOfRangeException(nameof(format), $"Unknown export format '{format}'.")
_
=> throw new ArgumentOutOfRangeException(
nameof(format),
$"Unknown export format '{format}'."
)
};
}
}
@@ -22,15 +22,20 @@ internal abstract class MessageWriter : IAsyncDisposable
Context = context;
}
public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
default;
public virtual ValueTask WriteMessageAsync(Message message, CancellationToken cancellationToken = default)
public virtual ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default
)
{
MessagesWritten++;
return default;
}
public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) =>
default;
public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync();
}
}
@@ -8,4 +8,4 @@ internal class FileSizePartitionLimit : PartitionLimit
public override bool IsReached(long messagesWritten, long bytesWritten) =>
bytesWritten >= _limit;
}
}
@@ -8,4 +8,4 @@ internal class MessageCountPartitionLimit : PartitionLimit
public override bool IsReached(long messagesWritten, long bytesWritten) =>
messagesWritten >= _limit;
}
}
@@ -3,4 +3,4 @@
internal class NullPartitionLimit : PartitionLimit
{
public override bool IsReached(long messagesWritten, long bytesWritten) => false;
}
}
@@ -18,11 +18,14 @@ public partial class PartitionLimit
var match = Regex.Match(value, @"^\s*(\d+[\.,]?\d*)\s*(\w)?b\s*$", RegexOptions.IgnoreCase);
// Number part
if (!double.TryParse(
if (
!double.TryParse(
match.Groups[1].Value,
NumberStyles.Float,
formatProvider,
out var number))
out var number
)
)
{
return null;
}
@@ -42,7 +45,7 @@ public partial class PartitionLimit
return null;
}
return (long) (number * magnitude);
return (long)(number * magnitude);
}
public static PartitionLimit? TryParse(string value, IFormatProvider? formatProvider = null)
@@ -58,5 +61,6 @@ public partial class PartitionLimit
}
public static PartitionLimit Parse(string value, IFormatProvider? formatProvider = null) =>
TryParse(value, formatProvider) ?? throw new FormatException($"Invalid partition limit '{value}'.");
}
TryParse(value, formatProvider)
?? throw new FormatException($"Invalid partition limit '{value}'.");
}
@@ -21,7 +21,8 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(text.Text);
return default;
@@ -29,19 +30,18 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name
);
_buffer.Append(emoji.IsCustomEmoji ? $":{emoji.Name}:" : emoji.Name);
return default;
}
protected override async ValueTask VisitMentionAsync(MentionNode mention,
CancellationToken cancellationToken = default)
protected override async ValueTask VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default
)
{
if (mention.Kind == MentionKind.Everyone)
{
@@ -86,7 +86,8 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
protected override ValueTask VisitTimestampAsync(
TimestampNode timestamp,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
_buffer.Append(
timestamp.Instant is not null
@@ -105,7 +106,8 @@ internal partial class PlainTextMarkdownVisitor
public static async ValueTask<string> FormatAsync(
ExportContext context,
string markdown,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var nodes = MarkdownParser.ParseMinimal(markdown);
@@ -114,4 +116,4 @@ internal partial class PlainTextMarkdownVisitor
return buffer.ToString();
}
}
}
@@ -7,20 +7,23 @@ namespace DiscordChatExporter.Core.Exporting;
internal static class PlainTextMessageExtensions
{
public static string GetFallbackContent(this Message message) => message.Kind switch
{
MessageKind.RecipientAdd => message.MentionedUsers.Any()
? $"Added {message.MentionedUsers.First().DisplayName} to the group."
: "Added a recipient.",
public static string GetFallbackContent(this Message message) =>
message.Kind switch
{
MessageKind.RecipientAdd
=> message.MentionedUsers.Any()
? $"Added {message.MentionedUsers.First().DisplayName} to the group."
: "Added a recipient.",
MessageKind.RecipientRemove => message.MentionedUsers.Any()
? message.Author.Id == message.MentionedUsers.First().Id
? "Left the group."
: $"Removed {message.MentionedUsers.First().DisplayName} from the group."
: "Removed a recipient.",
MessageKind.RecipientRemove
=> message.MentionedUsers.Any()
? message.Author.Id == message.MentionedUsers.First().Id
? "Left the group."
: $"Removed {message.MentionedUsers.First().DisplayName} from the group."
: "Removed a recipient.",
MessageKind.Call =>
$"Started a call that lasted {
MessageKind.Call
=> $"Started a call that lasted {
message
.CallEndedTimestamp?
.Pipe(t => t - message.Timestamp)
@@ -28,16 +31,16 @@ internal static class PlainTextMessageExtensions
.ToString("n0", CultureInfo.InvariantCulture) ?? "0"
} minutes.",
MessageKind.ChannelNameChange =>
!string.IsNullOrWhiteSpace(message.Content)
? $"Changed the channel name: {message.Content}"
: "Changed the channel name.",
MessageKind.ChannelNameChange
=> !string.IsNullOrWhiteSpace(message.Content)
? $"Changed the channel name: {message.Content}"
: "Changed the channel name.",
MessageKind.ChannelIconChange => "Changed the channel icon.",
MessageKind.ChannelPinnedMessage => "Pinned a message.",
MessageKind.ThreadCreated => "Started a thread.",
MessageKind.GuildMemberJoin => "Joined the server.",
MessageKind.ChannelIconChange => "Changed the channel icon.",
MessageKind.ChannelPinnedMessage => "Pinned a message.",
MessageKind.ThreadCreated => "Started a thread.",
MessageKind.GuildMemberJoin => "Joined the server.",
_ => message.Content
};
}
_ => message.Content
};
}
@@ -20,7 +20,8 @@ internal class PlainTextMessageWriter : MessageWriter
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
@@ -40,7 +41,8 @@ internal class PlainTextMessageWriter : MessageWriter
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (!attachments.Any())
return;
@@ -61,7 +63,8 @@ internal class PlainTextMessageWriter : MessageWriter
private async ValueTask WriteEmbedsAsync(
IReadOnlyList<Embed> embeds,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
foreach (var embed in embeds)
{
@@ -144,7 +147,8 @@ internal class PlainTextMessageWriter : MessageWriter
private async ValueTask WriteStickersAsync(
IReadOnlyList<Sticker> stickers,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (!stickers.Any())
return;
@@ -165,7 +169,8 @@ internal class PlainTextMessageWriter : MessageWriter
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (!reactions.Any())
return;
@@ -189,11 +194,15 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteLineAsync();
}
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(new string('=', 62));
await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}");
await _writer.WriteLineAsync($"Channel: {Context.Request.Channel.Category} / {Context.Request.Channel.Name}");
await _writer.WriteLineAsync(
$"Channel: {Context.Request.Channel.Category} / {Context.Request.Channel.Name}"
);
if (!string.IsNullOrWhiteSpace(Context.Request.Channel.Topic))
{
@@ -202,12 +211,16 @@ internal class PlainTextMessageWriter : MessageWriter
if (Context.Request.After is not null)
{
await _writer.WriteLineAsync($"After: {Context.FormatDate(Context.Request.After.Value.ToDate())}");
await _writer.WriteLineAsync(
$"After: {Context.FormatDate(Context.Request.After.Value.ToDate())}"
);
}
if (Context.Request.Before is not null)
{
await _writer.WriteLineAsync($"Before: {Context.FormatDate(Context.Request.Before.Value.ToDate())}");
await _writer.WriteLineAsync(
$"Before: {Context.FormatDate(Context.Request.Before.Value.ToDate())}"
);
}
await _writer.WriteLineAsync(new string('=', 62));
@@ -216,7 +229,8 @@ internal class PlainTextMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -246,7 +260,9 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteLineAsync();
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
public override async ValueTask WritePostambleAsync(
CancellationToken cancellationToken = default
)
{
await _writer.WriteLineAsync(new string('=', 62));
await _writer.WriteLineAsync($"Exported {MessagesWritten:N0} message(s)");
@@ -258,4 +274,4 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
}
@@ -8,17 +8,14 @@ internal record EmojiNode(
Snowflake? Id,
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated) : MarkdownNode
bool IsAnimated
) : MarkdownNode
{
public bool IsCustomEmoji => Id is not null;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => IsCustomEmoji
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public string Code => IsCustomEmoji ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
public EmojiNode(string name)
: this(null, name, false)
{
}
}
: this(null, name, false) { }
}
@@ -8,4 +8,4 @@ internal enum FormattingKind
Strikethrough,
Spoiler,
Quote
}
}
@@ -2,7 +2,6 @@
namespace DiscordChatExporter.Core.Markdown;
internal record FormattingNode(
FormattingKind Kind,
IReadOnlyList<MarkdownNode> Children
) : MarkdownNode, IContainerNode;
internal record FormattingNode(FormattingKind Kind, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode;
@@ -2,7 +2,6 @@
namespace DiscordChatExporter.Core.Markdown;
internal record HeadingNode(
int Level,
IReadOnlyList<MarkdownNode> Children
) : MarkdownNode, IContainerNode;
internal record HeadingNode(int Level, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode;
@@ -5,4 +5,4 @@ namespace DiscordChatExporter.Core.Markdown;
internal interface IContainerNode
{
IReadOnlyList<MarkdownNode> Children { get; }
}
}
@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record InlineCodeBlockNode(string Code) : MarkdownNode;
internal record InlineCodeBlockNode(string Code) : MarkdownNode;
@@ -3,12 +3,10 @@
namespace DiscordChatExporter.Core.Markdown;
// Named links can contain child nodes (e.g. [**bold URL**](https://test.com))
internal record LinkNode(
string Url,
IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode
internal record LinkNode(string Url, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode
{
public LinkNode(string url)
: this(url, new[] { new TextNode(url) })
{
}
}
: this(url, new[] { new TextNode(url) }) { }
}
@@ -2,4 +2,4 @@
namespace DiscordChatExporter.Core.Markdown;
internal record ListItemNode(IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode;
internal record ListItemNode(IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode;
@@ -2,4 +2,4 @@
namespace DiscordChatExporter.Core.Markdown;
internal record ListNode(IReadOnlyList<ListItemNode> Items) : MarkdownNode;
internal record ListNode(IReadOnlyList<ListItemNode> Items) : MarkdownNode;
@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal abstract record MarkdownNode;
internal abstract record MarkdownNode;
@@ -7,4 +7,4 @@ internal enum MentionKind
User,
Channel,
Role
}
}
@@ -3,4 +3,4 @@
namespace DiscordChatExporter.Core.Markdown;
// Null ID means it's a meta mention or an invalid mention
internal record MentionNode(Snowflake? TargetId, MentionKind Kind) : MarkdownNode;
internal record MentionNode(Snowflake? TargetId, MentionKind Kind) : MarkdownNode;
@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record MultiLineCodeBlockNode(string Language, string Code) : MarkdownNode;
internal record MultiLineCodeBlockNode(string Language, string Code) : MarkdownNode;
@@ -12,9 +12,7 @@ internal class AggregateMatcher<T> : IMatcher<T>
}
public AggregateMatcher(params IMatcher<T>[] matchers)
: this((IReadOnlyList<IMatcher<T>>) matchers)
{
}
: this((IReadOnlyList<IMatcher<T>>)matchers) { }
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
@@ -31,7 +29,9 @@ internal class AggregateMatcher<T> : IMatcher<T>
continue;
// If this match is earlier than previous earliest - replace
if (earliestMatch is null || match.Segment.StartIndex < earliestMatch.Segment.StartIndex)
if (
earliestMatch is null || match.Segment.StartIndex < earliestMatch.Segment.StartIndex
)
earliestMatch = match;
// If the earliest match starts at the very beginning - break,
@@ -42,4 +42,4 @@ internal class AggregateMatcher<T> : IMatcher<T>
return earliestMatch;
}
}
}
@@ -13,7 +13,8 @@ internal static class MatcherExtensions
public static IEnumerable<ParsedMatch<T>> MatchAll<T>(
this IMatcher<T> matcher,
StringSegment segment,
Func<StringSegment, T> transformFallback)
Func<StringSegment, T> transformFallback
)
{
// Loop through segments divided by individual matches
var currentIndex = segment.StartIndex;
@@ -21,10 +22,7 @@ internal static class MatcherExtensions
{
// Find a match within this segment
var match = matcher.TryMatch(
segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
)
segment.Relocate(currentIndex, segment.EndIndex - currentIndex)
);
if (match is null)
@@ -38,7 +36,10 @@ internal static class MatcherExtensions
match.Segment.StartIndex - currentIndex
);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
yield return new ParsedMatch<T>(
fallbackSegment,
transformFallback(fallbackSegment)
);
}
yield return match;
@@ -50,12 +51,9 @@ internal static class MatcherExtensions
// If EOL hasn't been reached - transform and yield remaining part as fallback
if (currentIndex < segment.EndIndex)
{
var fallbackSegment = segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
);
var fallbackSegment = segment.Relocate(currentIndex, segment.EndIndex - currentIndex);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
}
}
}
}
@@ -16,302 +16,354 @@ namespace DiscordChatExporter.Core.Markdown.Parsing;
internal static partial class MarkdownParser
{
private const RegexOptions DefaultRegexOptions =
RegexOptions.Compiled |
RegexOptions.IgnorePatternWhitespace |
RegexOptions.CultureInvariant |
RegexOptions.Multiline;
RegexOptions.Compiled
| RegexOptions.IgnorePatternWhitespace
| RegexOptions.CultureInvariant
| RegexOptions.Multiline;
/* Formatting */
private static readonly IMatcher<MarkdownNode> BoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly two closing asterisks.
new Regex(@"\*\*(.+?)\*\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Bold, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> BoldFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly two closing asterisks.
new Regex(@"\*\*(.+?)\*\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Bold, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly one closing asterisk.
// Opening asterisk must not be followed by whitespace.
// Closing asterisk must not be preceded by whitespace.
new Regex(@"\*(?!\s)(.+?)(?<!\s|\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly one closing asterisk.
// Opening asterisk must not be followed by whitespace.
// Closing asterisk must not be preceded by whitespace.
new Regex(
@"\*(?!\s)(.+?)(?<!\s|\*)\*(?!\*)",
DefaultRegexOptions | RegexOptions.Singleline
),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly three closing asterisks.
new Regex(@"\*(\*\*.+?\*\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1]), BoldFormattingNodeMatcher))
);
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly three closing asterisks.
new Regex(@"\*(\*\*.+?\*\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) =>
new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), BoldFormattingNodeMatcher)
)
);
private static readonly IMatcher<MarkdownNode> ItalicAltFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// Closing underscore must not be followed by a word character.
new Regex(@"_(.+?)_(?!\w)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicAltFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Closing underscore must not be followed by a word character.
new Regex(@"_(.+?)_(?!\w)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> UnderlineFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly two closing underscores.
new Regex(@"__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Underline, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> UnderlineFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly two closing underscores.
new Regex(@"__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Underline, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly three closing underscores.
new Regex(@"_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), UnderlineFormattingNodeMatcher)
)
(s, m) =>
new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), UnderlineFormattingNodeMatcher)
)
);
private static readonly IMatcher<MarkdownNode> StrikethroughFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Strikethrough, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> StrikethroughFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) =>
new FormattingNode(FormattingKind.Strikethrough, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SpoilerFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"\|\|(.+?)\|\|", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Spoiler, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SpoilerFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"\|\|(.+?)\|\|", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Spoiler, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
// Include the linebreak in the content so that the lines are preserved in quotes.
new Regex(@"^>\s(.+\n?)", DefaultRegexOptions),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Include the linebreak in the content so that the lines are preserved in quotes.
new Regex(@"^>\s(.+\n?)", DefaultRegexOptions),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
// Include the linebreaks in the content, so that the lines are preserved in quotes.
// Empty content is allowed within quotes.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1115
new Regex(@"(?:^>\s(.*\n?)){2,}", DefaultRegexOptions),
(s, m) => new FormattingNode(
FormattingKind.Quote,
m.Groups[1].Captures.SelectMany(c => Parse(s.Relocate(c))).ToArray()
)
);
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Include the linebreaks in the content, so that the lines are preserved in quotes.
// Empty content is allowed within quotes.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1115
new Regex(@"(?:^>\s(.*\n?)){2,}", DefaultRegexOptions),
(s, m) =>
new FormattingNode(
FormattingKind.Quote,
m.Groups[1].Captures.SelectMany(c => Parse(s.Relocate(c))).ToArray()
)
);
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"^>>>\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"^>>>\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> HeadingNodeMatcher = new RegexMatcher<MarkdownNode>(
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\#{1,3})\s(.+)\n", DefaultRegexOptions),
(s, m) => new HeadingNode(m.Groups[1].Length, Parse(s.Relocate(m.Groups[2])))
);
private static readonly IMatcher<MarkdownNode> HeadingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\#{1,3})\s(.+)\n", DefaultRegexOptions),
(s, m) => new HeadingNode(m.Groups[1].Length, Parse(s.Relocate(m.Groups[2])))
);
private static readonly IMatcher<MarkdownNode> ListNodeMatcher = new RegexMatcher<MarkdownNode>(
// Can be preceded by whitespace, which specifies the list's nesting level.
// Following lines that start with (level+1) whitespace are considered part of the list item.
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\s*)(?:[\-\*]\s(.+(?:\n\s\1.*)*)?\n?)+", DefaultRegexOptions),
(s, m) => new ListNode(
m.Groups[2].Captures.Select(c => new ListItemNode(Parse(s.Relocate(c)))).ToArray()
)
(s, m) =>
new ListNode(
m.Groups[2].Captures.Select(c => new ListItemNode(Parse(s.Relocate(c)))).ToArray()
)
);
/* Code blocks */
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
// One or two backticks are allowed, but they must match on both sides.
new Regex(@"(`{1,2})([^`]+)\1", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new InlineCodeBlockNode(m.Groups[2].Value)
);
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher =
new RegexMatcher<MarkdownNode>(
// One or two backticks are allowed, but they must match on both sides.
new Regex(@"(`{1,2})([^`]+)\1", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new InlineCodeBlockNode(m.Groups[2].Value)
);
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
// Language identifier is one word immediately after opening backticks, followed immediately by a linebreak.
// Blank lines at the beginning and at the end of content are trimmed.
new Regex(@"```(?:(\w*)\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n'))
);
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Language identifier is one word immediately after opening backticks, followed immediately by a linebreak.
// Blank lines at the beginning and at the end of content are trimmed.
new Regex(@"```(?:(\w*)\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) =>
new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n'))
);
/* Mentions */
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode(null, MentionKind.Everyone)
);
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher =
new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode(null, MentionKind.Everyone)
);
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
_ => new MentionNode(null, MentionKind.Here)
);
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher =
new StringMatcher<MarkdownNode>("@here", _ => new MentionNode(null, MentionKind.Here));
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <@123456> or <@!123456>
new Regex(@"<@!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.User)
);
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <@123456> or <@!123456>
new Regex(@"<@!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.User)
);
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <#123456>
new Regex(@"<\#!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Channel)
);
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <#123456>
new Regex(@"<\#!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Channel)
);
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <@&123456>
new Regex(@"<@&(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Role)
);
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <@&123456>
new Regex(@"<@&(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Role)
);
/* Emoji */
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(
@"(" +
// Country flag emoji (two regional indicator surrogate pairs)
@"(?:\uD83C[\uDDE6-\uDDFF]){2}|" +
// Digit emoji (digit followed by enclosing mark)
@"\d\p{Me}|" +
// Surrogate pair
@"\p{Cs}{2}|" +
// Miscellaneous characters
@"[" +
@"\u2600-\u2604" +
@"\u260E\u2611" +
@"\u2614-\u2615" +
@"\u2618\u261D\u2620" +
@"\u2622-\u2623" +
@"\u2626\u262A" +
@"\u262E-\u262F" +
@"\u2638-\u263A" +
@"\u2640\u2642" +
@"\u2648-\u2653" +
@"\u265F-\u2660" +
@"\u2663" +
@"\u2665-\u2666" +
@"\u2668\u267B" +
@"\u267E-\u267F" +
@"\u2692-\u2697" +
@"\u2699" +
@"\u269B-\u269C" +
@"\u26A0-\u26A1" +
@"\u26A7" +
@"\u26AA-\u26AB" +
@"\u26B0-\u26B1" +
@"\u26BD-\u26BE" +
@"\u26C4-\u26C5" +
@"\u26C8" +
@"\u26CE-\u26CF" +
@"\u26D1" +
@"\u26D3-\u26D4" +
@"\u26E9-\u26EA" +
@"\u26F0-\u26F5" +
@"\u26F7-\u26FA" +
@"\u26FD" +
@"]" +
@")", DefaultRegexOptions),
(_, m) => new EmojiNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(
@"("
+
// Country flag emoji (two regional indicator surrogate pairs)
@"(?:\uD83C[\uDDE6-\uDDFF]){2}|"
+
// Digit emoji (digit followed by enclosing mark)
@"\d\p{Me}|"
+
// Surrogate pair
@"\p{Cs}{2}|"
+
// Miscellaneous characters
@"["
+ @"\u2600-\u2604"
+ @"\u260E\u2611"
+ @"\u2614-\u2615"
+ @"\u2618\u261D\u2620"
+ @"\u2622-\u2623"
+ @"\u2626\u262A"
+ @"\u262E-\u262F"
+ @"\u2638-\u263A"
+ @"\u2640\u2642"
+ @"\u2648-\u2653"
+ @"\u265F-\u2660"
+ @"\u2663"
+ @"\u2665-\u2666"
+ @"\u2668\u267B"
+ @"\u267E-\u267F"
+ @"\u2692-\u2697"
+ @"\u2699"
+ @"\u269B-\u269C"
+ @"\u26A0-\u26A1"
+ @"\u26A7"
+ @"\u26AA-\u26AB"
+ @"\u26B0-\u26B1"
+ @"\u26BD-\u26BE"
+ @"\u26C4-\u26C5"
+ @"\u26C8"
+ @"\u26CE-\u26CF"
+ @"\u26D1"
+ @"\u26D3-\u26D4"
+ @"\u26E9-\u26EA"
+ @"\u26F0-\u26F5"
+ @"\u26F7-\u26FA"
+ @"\u26FD"
+ @"]"
+ @")",
DefaultRegexOptions
),
(_, m) => new EmojiNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> CodedStandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture :thinking:
new Regex(@":([\w_]+):", DefaultRegexOptions),
(_, m) => EmojiIndex.TryGetName(m.Groups[1].Value)?.Pipe(n => new EmojiNode(n))
);
private static readonly IMatcher<MarkdownNode> CodedStandardEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture :thinking:
new Regex(@":([\w_]+):", DefaultRegexOptions),
(_, m) => EmojiIndex.TryGetName(m.Groups[1].Value)?.Pipe(n => new EmojiNode(n))
);
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <:lul:123456> or <a:lul:123456>
new Regex(@"<(a)?:(.+?):(\d+?)>", DefaultRegexOptions),
(_, m) => new EmojiNode(
Snowflake.TryParse(m.Groups[3].Value),
m.Groups[2].Value,
!string.IsNullOrWhiteSpace(m.Groups[1].Value)
)
);
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <:lul:123456> or <a:lul:123456>
new Regex(@"<(a)?:(.+?):(\d+?)>", DefaultRegexOptions),
(_, m) =>
new EmojiNode(
Snowflake.TryParse(m.Groups[3].Value),
m.Groups[2].Value,
!string.IsNullOrWhiteSpace(m.Groups[1].Value)
)
);
/* Links */
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Any non-whitespace character after http:// or https://
// until the last punctuation character or whitespace.
new Regex(@"(https?://\S*[^\.,:;""'\s])", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Any non-whitespace character after http:// or https://
// until the last punctuation character or whitespace.
new Regex(@"(https?://\S*[^\.,:;""'\s])", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Same as auto link but also surrounded by angular brackets
new Regex(@"<(https?://\S*[^\.,:;""'\s])>", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Same as auto link but also surrounded by angular brackets
new Regex(@"<(https?://\S*[^\.,:;""'\s])>", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> MaskedLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture [title](link)
new Regex(@"\[(.+?)\]\((.+?)\)", DefaultRegexOptions),
(s, m) => new LinkNode(m.Groups[2].Value, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> MaskedLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture [title](link)
new Regex(@"\[(.+?)\]\((.+?)\)", DefaultRegexOptions),
(s, m) => new LinkNode(m.Groups[2].Value, Parse(s.Relocate(m.Groups[1])))
);
/* Text */
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher = new StringMatcher<MarkdownNode>(
// Capture the shrug kaomoji.
// This escapes it from matching for formatting.
@"¯\_(ツ)_/¯",
s => new TextNode(s.ToString())
);
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher =
new StringMatcher<MarkdownNode>(
// Capture the shrug kaomoji.
// This escapes it from matching for formatting.
@"¯\_(ツ)_/¯",
s => new TextNode(s.ToString())
);
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture some specific emoji that don't get rendered.
// This escapes them from matching for emoji.
new Regex(@"([\u26A7\u2640\u2642\u2695\u267E\u00A9\u00AE\u2122])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture some specific emoji that don't get rendered.
// This escapes them from matching for emoji.
new Regex(@"([\u26A7\u2640\u2642\u2695\u267E\u00A9\u00AE\u2122])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture any "symbol/other" character or surrogate pair preceded by a backslash.
// This escapes them from matching for emoji.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/230
new Regex(@"\\(\p{So}|\p{Cs}{2})", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture any "symbol/other" character or surrogate pair preceded by a backslash.
// This escapes them from matching for emoji.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/230
new Regex(@"\\(\p{So}|\p{Cs}{2})", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash.
// This escapes them from matching for formatting or other tokens.
new Regex(@"\\([^a-zA-Z0-9\s])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash.
// This escapes them from matching for formatting or other tokens.
new Regex(@"\\([^a-zA-Z0-9\s])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
/* Misc */
private static readonly IMatcher<MarkdownNode> TimestampNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <t:12345678> or <t:12345678:R>
new Regex(@"<t:(-?\d+)(?::(\w))?>", DefaultRegexOptions),
(_, m) =>
{
try
private static readonly IMatcher<MarkdownNode> TimestampNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <t:12345678> or <t:12345678:R>
new Regex(@"<t:(-?\d+)(?::(\w))?>", DefaultRegexOptions),
(_, m) =>
{
var instant = DateTimeOffset.UnixEpoch + TimeSpan.FromSeconds(
long.Parse(m.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture)
);
var format = m.Groups[2].Value switch
try
{
"t" => "h:mm tt",
"T" => "h:mm:ss tt",
"d" => "MM/dd/yyyy",
"D" => "MMMM dd, yyyy",
"f" => "MMMM dd, yyyy h:mm tt",
"F" => "dddd, MMMM dd, yyyy h:mm tt",
// Relative format is ignored because it doesn't make much sense in a static export
_ => null
};
var instant =
DateTimeOffset.UnixEpoch
+ TimeSpan.FromSeconds(
long.Parse(
m.Groups[1].Value,
NumberStyles.Integer,
CultureInfo.InvariantCulture
)
);
return new TimestampNode(instant, format);
var format = m.Groups[2].Value switch
{
"t" => "h:mm tt",
"T" => "h:mm:ss tt",
"d" => "MM/dd/yyyy",
"D" => "MMMM dd, yyyy",
"f" => "MMMM dd, yyyy h:mm tt",
"F" => "dddd, MMMM dd, yyyy h:mm tt",
// Relative format is ignored because it doesn't make much sense in a static export
_ => null
};
return new TimestampNode(instant, format);
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
// https://github.com/Tyrrrz/DiscordChatExporter/issues/766
catch (Exception ex)
when (ex is FormatException or ArgumentOutOfRangeException or OverflowException)
{
// For invalid timestamps, Discord renders "Invalid Date" instead of ignoring the markdown
return TimestampNode.Invalid;
}
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
// https://github.com/Tyrrrz/DiscordChatExporter/issues/766
catch (Exception ex) when (ex is FormatException or ArgumentOutOfRangeException or OverflowException)
{
// For invalid timestamps, Discord renders "Invalid Date" instead of ignoring the markdown
return TimestampNode.Invalid;
}
}
);
);
// Matchers that have similar patterns are ordered from most specific to least specific
private static readonly IMatcher<MarkdownNode> NodeMatcher = new AggregateMatcher<MarkdownNode>(
@@ -320,7 +372,6 @@ internal static partial class MarkdownParser
IgnoredEmojiTextNodeMatcher,
EscapedSymbolTextNodeMatcher,
EscapedCharacterTextNodeMatcher,
// Formatting
ItalicBoldFormattingNodeMatcher,
ItalicUnderlineFormattingNodeMatcher,
@@ -335,53 +386,46 @@ internal static partial class MarkdownParser
SingleLineQuoteNodeMatcher,
HeadingNodeMatcher,
ListNodeMatcher,
// Code blocks
MultiLineCodeBlockNodeMatcher,
InlineCodeBlockNodeMatcher,
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
// Links
MaskedLinkNodeMatcher,
AutoLinkNodeMatcher,
HiddenLinkNodeMatcher,
// Emoji
StandardEmojiNodeMatcher,
CustomEmojiNodeMatcher,
CodedStandardEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
// Minimal set of matchers for non-multimedia formats (e.g. plain text)
private static readonly IMatcher<MarkdownNode> MinimalNodeMatcher = new AggregateMatcher<MarkdownNode>(
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
private static readonly IMatcher<MarkdownNode> MinimalNodeMatcher =
new AggregateMatcher<MarkdownNode>(
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
// Emoji
CustomEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
// Emoji
CustomEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
private static IReadOnlyList<MarkdownNode> Parse(StringSegment segment, IMatcher<MarkdownNode> matcher) =>
matcher
.MatchAll(segment, s => new TextNode(s.ToString()))
.Select(r => r.Value)
.ToArray();
private static IReadOnlyList<MarkdownNode> Parse(
StringSegment segment,
IMatcher<MarkdownNode> matcher
) => matcher.MatchAll(segment, s => new TextNode(s.ToString())).Select(r => r.Value).ToArray();
}
internal static partial class MarkdownParser
@@ -417,4 +461,4 @@ internal static partial class MarkdownParser
return links;
}
}
}
@@ -9,56 +9,63 @@ internal abstract class MarkdownVisitor
{
protected virtual ValueTask VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual async ValueTask VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default) =>
await VisitAsync(formatting.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(formatting.Children, cancellationToken);
protected virtual async ValueTask VisitHeadingAsync(
HeadingNode heading,
CancellationToken cancellationToken = default) =>
await VisitAsync(heading.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(heading.Children, cancellationToken);
protected virtual async ValueTask VisitListAsync(
ListNode list,
CancellationToken cancellationToken = default) =>
await VisitAsync(list.Items, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(list.Items, cancellationToken);
protected virtual async ValueTask VisitListItemAsync(
ListItemNode listItem,
CancellationToken cancellationToken = default) =>
await VisitAsync(listItem.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(listItem.Children, cancellationToken);
protected virtual ValueTask VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual async ValueTask VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default) =>
await VisitAsync(link.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(link.Children, cancellationToken);
protected virtual ValueTask VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitTimestampAsync(
TimestampNode timestamp,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
public async ValueTask VisitAsync(
MarkdownNode node,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (node is TextNode text)
{
@@ -131,9 +138,10 @@ internal abstract class MarkdownVisitor
public async ValueTask VisitAsync(
IEnumerable<MarkdownNode> nodes,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
foreach (var node in nodes)
await VisitAsync(node, cancellationToken);
}
}
}
@@ -11,4 +11,4 @@ internal class ParsedMatch<T>
Segment = segment;
Value = value;
}
}
}
@@ -31,8 +31,6 @@ internal class RegexMatcher<T> : IMatcher<T>
var segmentMatch = segment.Relocate(match);
var value = _transform(segmentMatch, match);
return value is not null
? new ParsedMatch<T>(segmentMatch, value)
: null;
return value is not null ? new ParsedMatch<T>(segmentMatch, value) : null;
}
}
}
@@ -8,7 +8,11 @@ internal class StringMatcher<T> : IMatcher<T>
private readonly StringComparison _comparison;
private readonly Func<StringSegment, T?> _transform;
public StringMatcher(string needle, StringComparison comparison, Func<StringSegment, T?> transform)
public StringMatcher(
string needle,
StringComparison comparison,
Func<StringSegment, T?> transform
)
{
_needle = needle;
_comparison = comparison;
@@ -16,21 +20,22 @@ internal class StringMatcher<T> : IMatcher<T>
}
public StringMatcher(string needle, Func<StringSegment, T> transform)
: this(needle, StringComparison.Ordinal, transform)
{
}
: this(needle, StringComparison.Ordinal, transform) { }
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
var index = segment.Source.IndexOf(_needle, segment.StartIndex, segment.Length, _comparison);
var index = segment.Source.IndexOf(
_needle,
segment.StartIndex,
segment.Length,
_comparison
);
if (index < 0)
return null;
var segmentMatch = segment.Relocate(index, _needle.Length);
var value = _transform(segmentMatch);
return value is not null
? new ParsedMatch<T>(segmentMatch, value)
: null;
return value is not null ? new ParsedMatch<T>(segmentMatch, value) : null;
}
}
}
@@ -7,13 +7,12 @@ internal readonly record struct StringSegment(string Source, int StartIndex, int
public int EndIndex => StartIndex + Length;
public StringSegment(string target)
: this(target, 0, target.Length)
{
}
: this(target, 0, target.Length) { }
public StringSegment Relocate(int newStartIndex, int newLength) => new(Source, newStartIndex, newLength);
public StringSegment Relocate(int newStartIndex, int newLength) =>
new(Source, newStartIndex, newLength);
public StringSegment Relocate(Capture capture) => Relocate(capture.Index, capture.Length);
public override string ToString() => Source.Substring(StartIndex, Length);
}
}
@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record TextNode(string Text) : MarkdownNode;
internal record TextNode(string Text) : MarkdownNode;
@@ -6,4 +6,4 @@ namespace DiscordChatExporter.Core.Markdown;
internal record TimestampNode(DateTimeOffset? Instant, string? Format) : MarkdownNode
{
public static TimestampNode Invalid { get; } = new(null, null);
}
}
@@ -7,7 +7,8 @@ namespace DiscordChatExporter.Core.Utils.Extensions;
public static class AsyncCollectionExtensions
{
private static async ValueTask<IReadOnlyList<T>> CollectAsync<T>(
this IAsyncEnumerable<T> asyncEnumerable)
this IAsyncEnumerable<T> asyncEnumerable
)
{
var list = new List<T>();
@@ -18,6 +19,6 @@ public static class AsyncCollectionExtensions
}
public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>(
this IAsyncEnumerable<T> asyncEnumerable) =>
asyncEnumerable.CollectAsync().GetAwaiter();
}
this IAsyncEnumerable<T> asyncEnumerable
) => asyncEnumerable.CollectAsync().GetAwaiter();
}
@@ -11,11 +11,9 @@ public static class BinaryExtensions
foreach (var b in data)
{
buffer.Append(
b.ToString(isUpperCase ? "X2" : "x2", CultureInfo.InvariantCulture)
);
buffer.Append(b.ToString(isUpperCase ? "X2" : "x2", CultureInfo.InvariantCulture));
}
return buffer.ToString();
}
}
}
@@ -16,7 +16,8 @@ public static class CollectionExtensions
yield return (o, i++);
}
public static IEnumerable<T> WhereNotNull<T>(this IEnumerable<T?> source) where T : class
public static IEnumerable<T> WhereNotNull<T>(this IEnumerable<T?> source)
where T : class
{
foreach (var o in source)
{
@@ -24,4 +25,4 @@ public static class CollectionExtensions
yield return o;
}
}
}
}
@@ -11,4 +11,4 @@ public static class ColorExtensions
public static int ToRgb(this Color color) => color.ToArgb() & 0xffffff;
public static string ToHex(this Color color) => $"#{color.R:X2}{color.G:X2}{color.B:X2}";
}
}
@@ -7,4 +7,4 @@ public static class DateExtensions
{
public static string ToLocalString(this DateTimeOffset instant, string format) =>
instant.ToLocalTime().ToString(format, CultureInfo.InvariantCulture);
}
}
@@ -24,8 +24,8 @@ public static class ExceptionExtensions
public static IReadOnlyList<Exception> GetSelfAndChildren(this Exception exception)
{
var children = new List<Exception> {exception};
var children = new List<Exception> { exception };
PopulateChildren(exception, children);
return children;
}
}
}
@@ -5,13 +5,12 @@ namespace DiscordChatExporter.Core.Utils.Extensions;
public static class GenericExtensions
{
public static TOut Pipe<TIn, TOut>(this TIn input, Func<TIn, TOut> transform) => transform(input);
public static TOut Pipe<TIn, TOut>(this TIn input, Func<TIn, TOut> transform) =>
transform(input);
public static T? NullIf<T>(this T value, Func<T, bool> predicate) where T : struct =>
!predicate(value)
? value
: null;
public static T? NullIf<T>(this T value, Func<T, bool> predicate)
where T : struct => !predicate(value) ? value : null;
public static T? NullIfDefault<T>(this T value) where T : struct =>
value.NullIf(v => EqualityComparer<T>.Default.Equals(v, default));
}
public static T? NullIfDefault<T>(this T value)
where T : struct => value.NullIf(v => EqualityComparer<T>.Default.Equals(v, default));
}
@@ -5,7 +5,5 @@ namespace DiscordChatExporter.Core.Utils.Extensions;
public static class HttpExtensions
{
public static string? TryGetValue(this HttpHeaders headers, string name) =>
headers.TryGetValues(name, out var values)
? string.Concat(values)
: null;
}
headers.TryGetValues(name, out var values) ? string.Concat(values) : null;
}
@@ -7,14 +7,10 @@ namespace DiscordChatExporter.Core.Utils.Extensions;
public static class StringExtensions
{
public static string? NullIfWhiteSpace(this string str) =>
!string.IsNullOrWhiteSpace(str)
? str
: null;
!string.IsNullOrWhiteSpace(str) ? str : null;
public static string Truncate(this string str, int charCount) =>
str.Length > charCount
? str[..charCount]
: str;
str.Length > charCount ? str[..charCount] : str;
public static string ToSpaceSeparatedWords(this string str)
{
@@ -41,13 +37,9 @@ public static class StringExtensions
}
}
public static T? ParseEnumOrNull<T>(this string str, bool ignoreCase = true) where T : struct, Enum =>
Enum.TryParse<T>(str, ignoreCase, out var result)
? result
: null;
public static T? ParseEnumOrNull<T>(this string str, bool ignoreCase = true)
where T : struct, Enum => Enum.TryParse<T>(str, ignoreCase, out var result) ? result : null;
public static StringBuilder AppendIfNotEmpty(this StringBuilder builder, char value) =>
builder.Length > 0
? builder.Append(value)
: builder;
}
builder.Length > 0 ? builder.Append(value) : builder;
}
@@ -16,11 +16,12 @@ public static class SuperpowerExtensions
// Only used for debugging while writing Superpower parsers.
// From https://twitter.com/nblumhardt/status/1389349059786264578
[ExcludeFromCodeCoverage]
public static TextParser<T> Log<T>(this TextParser<T> parser, string description) => i =>
{
Console.WriteLine($"Trying {description} ->");
var r = parser(i);
Console.WriteLine($"Result was {r}");
return r;
};
}
public static TextParser<T> Log<T>(this TextParser<T> parser, string description) =>
i =>
{
Console.WriteLine($"Trying {description} ->");
var r = parser(i);
Console.WriteLine($"Result was {r}");
return r;
};
}
@@ -14,4 +14,4 @@ public static class TimeSpanExtensions
return value;
}
}
}

Some files were not shown because too many files have changed in this diff Show More