calvin.erfmann/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-04-29 01:07:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use CSharpier

Browse Source
This commit is contained in:
Tyrrrz
2023-08-22 21:17:19 +03:00
parent c410e745b1
commit 20f58963a6
174 changed files with 11084 additions and 10670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
DiscordChatExporter.Core/Markdown/EmojiNode.cs
View File

@@ -8,17 +8,14 @@ internal record EmojiNode(
Snowflake? Id,
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated) : MarkdownNode
bool IsAnimated
) : MarkdownNode
{
public bool IsCustomEmoji => Id is not null;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => IsCustomEmoji
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public string Code => IsCustomEmoji ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
public EmojiNode(string name)
: this(null, name, false)
{
}
}
: this(null, name, false) { }
}

2
DiscordChatExporter.Core/Markdown/FormattingKind.cs
View File

@@ -8,4 +8,4 @@ internal enum FormattingKind
Strikethrough,
Spoiler,
Quote
}
}

7
DiscordChatExporter.Core/Markdown/FormattingNode.cs
View File

@@ -2,7 +2,6 @@
namespace DiscordChatExporter.Core.Markdown;
internal record FormattingNode(
FormattingKind Kind,
IReadOnlyList<MarkdownNode> Children
) : MarkdownNode, IContainerNode;
internal record FormattingNode(FormattingKind Kind, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode;

7
DiscordChatExporter.Core/Markdown/HeadingNode.cs
View File

@@ -2,7 +2,6 @@
namespace DiscordChatExporter.Core.Markdown;
internal record HeadingNode(
int Level,
IReadOnlyList<MarkdownNode> Children
) : MarkdownNode, IContainerNode;
internal record HeadingNode(int Level, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode;

2
DiscordChatExporter.Core/Markdown/IContainerNode.cs
View File

@@ -5,4 +5,4 @@ namespace DiscordChatExporter.Core.Markdown;
internal interface IContainerNode
{
IReadOnlyList<MarkdownNode> Children { get; }
}
}

2
DiscordChatExporter.Core/Markdown/InlineCodeBlockNode.cs
View File

@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record InlineCodeBlockNode(string Code) : MarkdownNode;
internal record InlineCodeBlockNode(string Code) : MarkdownNode;

12
DiscordChatExporter.Core/Markdown/LinkNode.cs
View File

@@ -3,12 +3,10 @@
namespace DiscordChatExporter.Core.Markdown;
// Named links can contain child nodes (e.g. [**bold URL**](https://test.com))
internal record LinkNode(
string Url,
IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode
internal record LinkNode(string Url, IReadOnlyList<MarkdownNode> Children)
: MarkdownNode,
IContainerNode
{
public LinkNode(string url)
: this(url, new[] { new TextNode(url) })
{
}
}
: this(url, new[] { new TextNode(url) }) { }
}

2
DiscordChatExporter.Core/Markdown/ListItemNode.cs
View File

@@ -2,4 +2,4 @@
namespace DiscordChatExporter.Core.Markdown;
internal record ListItemNode(IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode;
internal record ListItemNode(IReadOnlyList<MarkdownNode> Children) : MarkdownNode, IContainerNode;

2
DiscordChatExporter.Core/Markdown/ListNode.cs
View File

@@ -2,4 +2,4 @@
namespace DiscordChatExporter.Core.Markdown;
internal record ListNode(IReadOnlyList<ListItemNode> Items) : MarkdownNode;
internal record ListNode(IReadOnlyList<ListItemNode> Items) : MarkdownNode;

2
DiscordChatExporter.Core/Markdown/MarkdownNode.cs
View File

@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal abstract record MarkdownNode;
internal abstract record MarkdownNode;

2
DiscordChatExporter.Core/Markdown/MentionKind.cs
View File

@@ -7,4 +7,4 @@ internal enum MentionKind
User,
Channel,
Role
}
}

2
DiscordChatExporter.Core/Markdown/MentionNode.cs
View File

@@ -3,4 +3,4 @@
namespace DiscordChatExporter.Core.Markdown;
// Null ID means it's a meta mention or an invalid mention
internal record MentionNode(Snowflake? TargetId, MentionKind Kind) : MarkdownNode;
internal record MentionNode(Snowflake? TargetId, MentionKind Kind) : MarkdownNode;

2
DiscordChatExporter.Core/Markdown/MultiLineCodeBlockNode.cs
View File

@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record MultiLineCodeBlockNode(string Language, string Code) : MarkdownNode;
internal record MultiLineCodeBlockNode(string Language, string Code) : MarkdownNode;

10
DiscordChatExporter.Core/Markdown/Parsing/AggregateMatcher.cs
View File

@@ -12,9 +12,7 @@ internal class AggregateMatcher<T> : IMatcher<T>
}
public AggregateMatcher(params IMatcher<T>[] matchers)
: this((IReadOnlyList<IMatcher<T>>) matchers)
{
}
: this((IReadOnlyList<IMatcher<T>>)matchers) { }
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
@@ -31,7 +29,9 @@ internal class AggregateMatcher<T> : IMatcher<T>
continue;
// If this match is earlier than previous earliest - replace
if (earliestMatch is null || match.Segment.StartIndex < earliestMatch.Segment.StartIndex)
if (
earliestMatch is null || match.Segment.StartIndex < earliestMatch.Segment.StartIndex
)
earliestMatch = match;
// If the earliest match starts at the very beginning - break,
@@ -42,4 +42,4 @@ internal class AggregateMatcher<T> : IMatcher<T>
return earliestMatch;
}
}
}

20
DiscordChatExporter.Core/Markdown/Parsing/IMatcher.cs
View File

@@ -13,7 +13,8 @@ internal static class MatcherExtensions
public static IEnumerable<ParsedMatch<T>> MatchAll<T>(
this IMatcher<T> matcher,
StringSegment segment,
Func<StringSegment, T> transformFallback)
Func<StringSegment, T> transformFallback
)
{
// Loop through segments divided by individual matches
var currentIndex = segment.StartIndex;
@@ -21,10 +22,7 @@ internal static class MatcherExtensions
{
// Find a match within this segment
var match = matcher.TryMatch(
segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
)
segment.Relocate(currentIndex, segment.EndIndex - currentIndex)
);
if (match is null)
@@ -38,7 +36,10 @@ internal static class MatcherExtensions
match.Segment.StartIndex - currentIndex
);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
yield return new ParsedMatch<T>(
fallbackSegment,
transformFallback(fallbackSegment)
);
}
yield return match;
@@ -50,12 +51,9 @@ internal static class MatcherExtensions
// If EOL hasn't been reached - transform and yield remaining part as fallback
if (currentIndex < segment.EndIndex)
{
var fallbackSegment = segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
);
var fallbackSegment = segment.Relocate(currentIndex, segment.EndIndex - currentIndex);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
}
}
}
}

568
DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs
View File

@@ -16,302 +16,354 @@ namespace DiscordChatExporter.Core.Markdown.Parsing;
internal static partial class MarkdownParser
{
private const RegexOptions DefaultRegexOptions =
RegexOptions.Compiled |
RegexOptions.IgnorePatternWhitespace |
RegexOptions.CultureInvariant |
RegexOptions.Multiline;
RegexOptions.Compiled
| RegexOptions.IgnorePatternWhitespace
| RegexOptions.CultureInvariant
| RegexOptions.Multiline;
/* Formatting */
private static readonly IMatcher<MarkdownNode> BoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly two closing asterisks.
new Regex(@"\*\*(.+?)\*\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Bold, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> BoldFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly two closing asterisks.
new Regex(@"\*\*(.+?)\*\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Bold, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly one closing asterisk.
// Opening asterisk must not be followed by whitespace.
// Closing asterisk must not be preceded by whitespace.
new Regex(@"\*(?!\s)(.+?)(?<!\s|\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly one closing asterisk.
// Opening asterisk must not be followed by whitespace.
// Closing asterisk must not be preceded by whitespace.
new Regex(
@"\*(?!\s)(.+?)(?<!\s|\*)\*(?!\*)",
DefaultRegexOptions | RegexOptions.Singleline
),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly three closing asterisks.
new Regex(@"\*(\*\*.+?\*\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1]), BoldFormattingNodeMatcher))
);
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly three closing asterisks.
new Regex(@"\*(\*\*.+?\*\*)\*(?!\*)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) =>
new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), BoldFormattingNodeMatcher)
)
);
private static readonly IMatcher<MarkdownNode> ItalicAltFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// Closing underscore must not be followed by a word character.
new Regex(@"_(.+?)_(?!\w)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicAltFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Closing underscore must not be followed by a word character.
new Regex(@"_(.+?)_(?!\w)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> UnderlineFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
// There must be exactly two closing underscores.
new Regex(@"__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Underline, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> UnderlineFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly two closing underscores.
new Regex(@"__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Underline, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// There must be exactly three closing underscores.
new Regex(@"_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), UnderlineFormattingNodeMatcher)
)
(s, m) =>
new FormattingNode(
FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), UnderlineFormattingNodeMatcher)
)
);
private static readonly IMatcher<MarkdownNode> StrikethroughFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Strikethrough, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> StrikethroughFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) =>
new FormattingNode(FormattingKind.Strikethrough, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SpoilerFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"\|\|(.+?)\|\|", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Spoiler, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SpoilerFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"\|\|(.+?)\|\|", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Spoiler, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
// Include the linebreak in the content so that the lines are preserved in quotes.
new Regex(@"^>\s(.+\n?)", DefaultRegexOptions),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Include the linebreak in the content so that the lines are preserved in quotes.
new Regex(@"^>\s(.+\n?)", DefaultRegexOptions),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
// Include the linebreaks in the content, so that the lines are preserved in quotes.
// Empty content is allowed within quotes.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1115
new Regex(@"(?:^>\s(.*\n?)){2,}", DefaultRegexOptions),
(s, m) => new FormattingNode(
FormattingKind.Quote,
m.Groups[1].Captures.SelectMany(c => Parse(s.Relocate(c))).ToArray()
)
);
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Include the linebreaks in the content, so that the lines are preserved in quotes.
// Empty content is allowed within quotes.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1115
new Regex(@"(?:^>\s(.*\n?)){2,}", DefaultRegexOptions),
(s, m) =>
new FormattingNode(
FormattingKind.Quote,
m.Groups[1].Captures.SelectMany(c => Parse(s.Relocate(c))).ToArray()
)
);
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(@"^>>>\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(@"^>>>\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> HeadingNodeMatcher = new RegexMatcher<MarkdownNode>(
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\#{1,3})\s(.+)\n", DefaultRegexOptions),
(s, m) => new HeadingNode(m.Groups[1].Length, Parse(s.Relocate(m.Groups[2])))
);
private static readonly IMatcher<MarkdownNode> HeadingNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\#{1,3})\s(.+)\n", DefaultRegexOptions),
(s, m) => new HeadingNode(m.Groups[1].Length, Parse(s.Relocate(m.Groups[2])))
);
private static readonly IMatcher<MarkdownNode> ListNodeMatcher = new RegexMatcher<MarkdownNode>(
// Can be preceded by whitespace, which specifies the list's nesting level.
// Following lines that start with (level+1) whitespace are considered part of the list item.
// Consume the linebreak so that it's not attached to following nodes.
new Regex(@"^(\s*)(?:[\-\*]\s(.+(?:\n\s\1.*)*)?\n?)+", DefaultRegexOptions),
(s, m) => new ListNode(
m.Groups[2].Captures.Select(c => new ListItemNode(Parse(s.Relocate(c)))).ToArray()
)
(s, m) =>
new ListNode(
m.Groups[2].Captures.Select(c => new ListItemNode(Parse(s.Relocate(c)))).ToArray()
)
);
/* Code blocks */
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
// One or two backticks are allowed, but they must match on both sides.
new Regex(@"(`{1,2})([^`]+)\1", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new InlineCodeBlockNode(m.Groups[2].Value)
);
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher =
new RegexMatcher<MarkdownNode>(
// One or two backticks are allowed, but they must match on both sides.
new Regex(@"(`{1,2})([^`]+)\1", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new InlineCodeBlockNode(m.Groups[2].Value)
);
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
// Language identifier is one word immediately after opening backticks, followed immediately by a linebreak.
// Blank lines at the beginning and at the end of content are trimmed.
new Regex(@"```(?:(\w*)\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) => new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n'))
);
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Language identifier is one word immediately after opening backticks, followed immediately by a linebreak.
// Blank lines at the beginning and at the end of content are trimmed.
new Regex(@"```(?:(\w*)\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
(_, m) =>
new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n'))
);
/* Mentions */
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode(null, MentionKind.Everyone)
);
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher =
new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode(null, MentionKind.Everyone)
);
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
_ => new MentionNode(null, MentionKind.Here)
);
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher =
new StringMatcher<MarkdownNode>("@here", _ => new MentionNode(null, MentionKind.Here));
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <@123456> or <@!123456>
new Regex(@"<@!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.User)
);
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <@123456> or <@!123456>
new Regex(@"<@!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.User)
);
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <#123456>
new Regex(@"<\#!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Channel)
);
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <#123456>
new Regex(@"<\#!?(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Channel)
);
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <@&123456>
new Regex(@"<@&(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Role)
);
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <@&123456>
new Regex(@"<@&(\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Role)
);
/* Emoji */
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(
@"(" +
// Country flag emoji (two regional indicator surrogate pairs)
@"(?:\uD83C[\uDDE6-\uDDFF]){2}|" +
// Digit emoji (digit followed by enclosing mark)
@"\d\p{Me}|" +
// Surrogate pair
@"\p{Cs}{2}|" +
// Miscellaneous characters
@"[" +
@"\u2600-\u2604" +
@"\u260E\u2611" +
@"\u2614-\u2615" +
@"\u2618\u261D\u2620" +
@"\u2622-\u2623" +
@"\u2626\u262A" +
@"\u262E-\u262F" +
@"\u2638-\u263A" +
@"\u2640\u2642" +
@"\u2648-\u2653" +
@"\u265F-\u2660" +
@"\u2663" +
@"\u2665-\u2666" +
@"\u2668\u267B" +
@"\u267E-\u267F" +
@"\u2692-\u2697" +
@"\u2699" +
@"\u269B-\u269C" +
@"\u26A0-\u26A1" +
@"\u26A7" +
@"\u26AA-\u26AB" +
@"\u26B0-\u26B1" +
@"\u26BD-\u26BE" +
@"\u26C4-\u26C5" +
@"\u26C8" +
@"\u26CE-\u26CF" +
@"\u26D1" +
@"\u26D3-\u26D4" +
@"\u26E9-\u26EA" +
@"\u26F0-\u26F5" +
@"\u26F7-\u26FA" +
@"\u26FD" +
@"]" +
@")", DefaultRegexOptions),
(_, m) => new EmojiNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex(
@"("
+
// Country flag emoji (two regional indicator surrogate pairs)
@"(?:\uD83C[\uDDE6-\uDDFF]){2}|"
+
// Digit emoji (digit followed by enclosing mark)
@"\d\p{Me}|"
+
// Surrogate pair
@"\p{Cs}{2}|"
+
// Miscellaneous characters
@"["
+ @"\u2600-\u2604"
+ @"\u260E\u2611"
+ @"\u2614-\u2615"
+ @"\u2618\u261D\u2620"
+ @"\u2622-\u2623"
+ @"\u2626\u262A"
+ @"\u262E-\u262F"
+ @"\u2638-\u263A"
+ @"\u2640\u2642"
+ @"\u2648-\u2653"
+ @"\u265F-\u2660"
+ @"\u2663"
+ @"\u2665-\u2666"
+ @"\u2668\u267B"
+ @"\u267E-\u267F"
+ @"\u2692-\u2697"
+ @"\u2699"
+ @"\u269B-\u269C"
+ @"\u26A0-\u26A1"
+ @"\u26A7"
+ @"\u26AA-\u26AB"
+ @"\u26B0-\u26B1"
+ @"\u26BD-\u26BE"
+ @"\u26C4-\u26C5"
+ @"\u26C8"
+ @"\u26CE-\u26CF"
+ @"\u26D1"
+ @"\u26D3-\u26D4"
+ @"\u26E9-\u26EA"
+ @"\u26F0-\u26F5"
+ @"\u26F7-\u26FA"
+ @"\u26FD"
+ @"]"
+ @")",
DefaultRegexOptions
),
(_, m) => new EmojiNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> CodedStandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture :thinking:
new Regex(@":([\w_]+):", DefaultRegexOptions),
(_, m) => EmojiIndex.TryGetName(m.Groups[1].Value)?.Pipe(n => new EmojiNode(n))
);
private static readonly IMatcher<MarkdownNode> CodedStandardEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture :thinking:
new Regex(@":([\w_]+):", DefaultRegexOptions),
(_, m) => EmojiIndex.TryGetName(m.Groups[1].Value)?.Pipe(n => new EmojiNode(n))
);
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <:lul:123456> or <a:lul:123456>
new Regex(@"<(a)?:(.+?):(\d+?)>", DefaultRegexOptions),
(_, m) => new EmojiNode(
Snowflake.TryParse(m.Groups[3].Value),
m.Groups[2].Value,
!string.IsNullOrWhiteSpace(m.Groups[1].Value)
)
);
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <:lul:123456> or <a:lul:123456>
new Regex(@"<(a)?:(.+?):(\d+?)>", DefaultRegexOptions),
(_, m) =>
new EmojiNode(
Snowflake.TryParse(m.Groups[3].Value),
m.Groups[2].Value,
!string.IsNullOrWhiteSpace(m.Groups[1].Value)
)
);
/* Links */
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Any non-whitespace character after http:// or https://
// until the last punctuation character or whitespace.
new Regex(@"(https?://\S*[^\.,:;""'\s])", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Any non-whitespace character after http:// or https://
// until the last punctuation character or whitespace.
new Regex(@"(https?://\S*[^\.,:;""'\s])", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Same as auto link but also surrounded by angular brackets
new Regex(@"<(https?://\S*[^\.,:;""'\s])>", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Same as auto link but also surrounded by angular brackets
new Regex(@"<(https?://\S*[^\.,:;""'\s])>", DefaultRegexOptions),
(_, m) => new LinkNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> MaskedLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture [title](link)
new Regex(@"\[(.+?)\]\((.+?)\)", DefaultRegexOptions),
(s, m) => new LinkNode(m.Groups[2].Value, Parse(s.Relocate(m.Groups[1])))
);
private static readonly IMatcher<MarkdownNode> MaskedLinkNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture [title](link)
new Regex(@"\[(.+?)\]\((.+?)\)", DefaultRegexOptions),
(s, m) => new LinkNode(m.Groups[2].Value, Parse(s.Relocate(m.Groups[1])))
);
/* Text */
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher = new StringMatcher<MarkdownNode>(
// Capture the shrug kaomoji.
// This escapes it from matching for formatting.
@"¯\_(ツ)_/¯",
s => new TextNode(s.ToString())
);
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher =
new StringMatcher<MarkdownNode>(
// Capture the shrug kaomoji.
// This escapes it from matching for formatting.
@"¯\_(ツ)_/¯",
s => new TextNode(s.ToString())
);
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture some specific emoji that don't get rendered.
// This escapes them from matching for emoji.
new Regex(@"([\u26A7\u2640\u2642\u2695\u267E\u00A9\u00AE\u2122])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture some specific emoji that don't get rendered.
// This escapes them from matching for emoji.
new Regex(@"([\u26A7\u2640\u2642\u2695\u267E\u00A9\u00AE\u2122])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture any "symbol/other" character or surrogate pair preceded by a backslash.
// This escapes them from matching for emoji.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/230
new Regex(@"\\(\p{So}|\p{Cs}{2})", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture any "symbol/other" character or surrogate pair preceded by a backslash.
// This escapes them from matching for emoji.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/230
new Regex(@"\\(\p{So}|\p{Cs}{2})", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash.
// This escapes them from matching for formatting or other tokens.
new Regex(@"\\([^a-zA-Z0-9\s])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash.
// This escapes them from matching for formatting or other tokens.
new Regex(@"\\([^a-zA-Z0-9\s])", DefaultRegexOptions),
(_, m) => new TextNode(m.Groups[1].Value)
);
/* Misc */
private static readonly IMatcher<MarkdownNode> TimestampNodeMatcher = new RegexMatcher<MarkdownNode>(
// Capture <t:12345678> or <t:12345678:R>
new Regex(@"<t:(-?\d+)(?::(\w))?>", DefaultRegexOptions),
(_, m) =>
{
try
private static readonly IMatcher<MarkdownNode> TimestampNodeMatcher =
new RegexMatcher<MarkdownNode>(
// Capture <t:12345678> or <t:12345678:R>
new Regex(@"<t:(-?\d+)(?::(\w))?>", DefaultRegexOptions),
(_, m) =>
{
var instant = DateTimeOffset.UnixEpoch + TimeSpan.FromSeconds(
long.Parse(m.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture)
);
var format = m.Groups[2].Value switch
try
{
"t" => "h:mm tt",
"T" => "h:mm:ss tt",
"d" => "MM/dd/yyyy",
"D" => "MMMM dd, yyyy",
"f" => "MMMM dd, yyyy h:mm tt",
"F" => "dddd, MMMM dd, yyyy h:mm tt",
// Relative format is ignored because it doesn't make much sense in a static export
_ => null
};
var instant =
DateTimeOffset.UnixEpoch
+ TimeSpan.FromSeconds(
long.Parse(
m.Groups[1].Value,
NumberStyles.Integer,
CultureInfo.InvariantCulture
)
);
return new TimestampNode(instant, format);
var format = m.Groups[2].Value switch
{
"t" => "h:mm tt",
"T" => "h:mm:ss tt",
"d" => "MM/dd/yyyy",
"D" => "MMMM dd, yyyy",
"f" => "MMMM dd, yyyy h:mm tt",
"F" => "dddd, MMMM dd, yyyy h:mm tt",
// Relative format is ignored because it doesn't make much sense in a static export
_ => null
};
return new TimestampNode(instant, format);
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
// https://github.com/Tyrrrz/DiscordChatExporter/issues/766
catch (Exception ex)
when (ex is FormatException or ArgumentOutOfRangeException or OverflowException)
{
// For invalid timestamps, Discord renders "Invalid Date" instead of ignoring the markdown
return TimestampNode.Invalid;
}
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
// https://github.com/Tyrrrz/DiscordChatExporter/issues/766
catch (Exception ex) when (ex is FormatException or ArgumentOutOfRangeException or OverflowException)
{
// For invalid timestamps, Discord renders "Invalid Date" instead of ignoring the markdown
return TimestampNode.Invalid;
}
}
);
);
// Matchers that have similar patterns are ordered from most specific to least specific
private static readonly IMatcher<MarkdownNode> NodeMatcher = new AggregateMatcher<MarkdownNode>(
@@ -320,7 +372,6 @@ internal static partial class MarkdownParser
IgnoredEmojiTextNodeMatcher,
EscapedSymbolTextNodeMatcher,
EscapedCharacterTextNodeMatcher,
// Formatting
ItalicBoldFormattingNodeMatcher,
ItalicUnderlineFormattingNodeMatcher,
@@ -335,53 +386,46 @@ internal static partial class MarkdownParser
SingleLineQuoteNodeMatcher,
HeadingNodeMatcher,
ListNodeMatcher,
// Code blocks
MultiLineCodeBlockNodeMatcher,
InlineCodeBlockNodeMatcher,
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
// Links
MaskedLinkNodeMatcher,
AutoLinkNodeMatcher,
HiddenLinkNodeMatcher,
// Emoji
StandardEmojiNodeMatcher,
CustomEmojiNodeMatcher,
CodedStandardEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
// Minimal set of matchers for non-multimedia formats (e.g. plain text)
private static readonly IMatcher<MarkdownNode> MinimalNodeMatcher = new AggregateMatcher<MarkdownNode>(
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
private static readonly IMatcher<MarkdownNode> MinimalNodeMatcher =
new AggregateMatcher<MarkdownNode>(
// Mentions
EveryoneMentionNodeMatcher,
HereMentionNodeMatcher,
UserMentionNodeMatcher,
ChannelMentionNodeMatcher,
RoleMentionNodeMatcher,
// Emoji
CustomEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
// Emoji
CustomEmojiNodeMatcher,
// Misc
TimestampNodeMatcher
);
private static IReadOnlyList<MarkdownNode> Parse(StringSegment segment, IMatcher<MarkdownNode> matcher) =>
matcher
.MatchAll(segment, s => new TextNode(s.ToString()))
.Select(r => r.Value)
.ToArray();
private static IReadOnlyList<MarkdownNode> Parse(
StringSegment segment,
IMatcher<MarkdownNode> matcher
) => matcher.MatchAll(segment, s => new TextNode(s.ToString())).Select(r => r.Value).ToArray();
}
internal static partial class MarkdownParser
@@ -417,4 +461,4 @@ internal static partial class MarkdownParser
return links;
}
}
}

46
DiscordChatExporter.Core/Markdown/Parsing/MarkdownVisitor.cs
View File

@@ -9,56 +9,63 @@ internal abstract class MarkdownVisitor
{
protected virtual ValueTask VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual async ValueTask VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default) =>
await VisitAsync(formatting.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(formatting.Children, cancellationToken);
protected virtual async ValueTask VisitHeadingAsync(
HeadingNode heading,
CancellationToken cancellationToken = default) =>
await VisitAsync(heading.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(heading.Children, cancellationToken);
protected virtual async ValueTask VisitListAsync(
ListNode list,
CancellationToken cancellationToken = default) =>
await VisitAsync(list.Items, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(list.Items, cancellationToken);
protected virtual async ValueTask VisitListItemAsync(
ListItemNode listItem,
CancellationToken cancellationToken = default) =>
await VisitAsync(listItem.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(listItem.Children, cancellationToken);
protected virtual ValueTask VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual async ValueTask VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default) =>
await VisitAsync(link.Children, cancellationToken);
CancellationToken cancellationToken = default
) => await VisitAsync(link.Children, cancellationToken);
protected virtual ValueTask VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
protected virtual ValueTask VisitTimestampAsync(
TimestampNode timestamp,
CancellationToken cancellationToken = default) => default;
CancellationToken cancellationToken = default
) => default;
public async ValueTask VisitAsync(
MarkdownNode node,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (node is TextNode text)
{
@@ -131,9 +138,10 @@ internal abstract class MarkdownVisitor
public async ValueTask VisitAsync(
IEnumerable<MarkdownNode> nodes,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
foreach (var node in nodes)
await VisitAsync(node, cancellationToken);
}
}
}

2
DiscordChatExporter.Core/Markdown/Parsing/ParsedMatch.cs
View File

@@ -11,4 +11,4 @@ internal class ParsedMatch<T>
Segment = segment;
Value = value;
}
}
}

6
DiscordChatExporter.Core/Markdown/Parsing/RegexMatcher.cs
View File

@@ -31,8 +31,6 @@ internal class RegexMatcher<T> : IMatcher<T>
var segmentMatch = segment.Relocate(match);
var value = _transform(segmentMatch, match);
return value is not null
? new ParsedMatch<T>(segmentMatch, value)
: null;
return value is not null ? new ParsedMatch<T>(segmentMatch, value) : null;
}
}
}

23
DiscordChatExporter.Core/Markdown/Parsing/StringMatcher.cs
View File

@@ -8,7 +8,11 @@ internal class StringMatcher<T> : IMatcher<T>
private readonly StringComparison _comparison;
private readonly Func<StringSegment, T?> _transform;
public StringMatcher(string needle, StringComparison comparison, Func<StringSegment, T?> transform)
public StringMatcher(
string needle,
StringComparison comparison,
Func<StringSegment, T?> transform
)
{
_needle = needle;
_comparison = comparison;
@@ -16,21 +20,22 @@ internal class StringMatcher<T> : IMatcher<T>
}
public StringMatcher(string needle, Func<StringSegment, T> transform)
: this(needle, StringComparison.Ordinal, transform)
{
}
: this(needle, StringComparison.Ordinal, transform) { }
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
var index = segment.Source.IndexOf(_needle, segment.StartIndex, segment.Length, _comparison);
var index = segment.Source.IndexOf(
_needle,
segment.StartIndex,
segment.Length,
_comparison
);
if (index < 0)
return null;
var segmentMatch = segment.Relocate(index, _needle.Length);
var value = _transform(segmentMatch);
return value is not null
? new ParsedMatch<T>(segmentMatch, value)
: null;
return value is not null ? new ParsedMatch<T>(segmentMatch, value) : null;
}
}
}

9
DiscordChatExporter.Core/Markdown/Parsing/StringSegment.cs
View File

@@ -7,13 +7,12 @@ internal readonly record struct StringSegment(string Source, int StartIndex, int
public int EndIndex => StartIndex + Length;
public StringSegment(string target)
: this(target, 0, target.Length)
{
}
: this(target, 0, target.Length) { }
public StringSegment Relocate(int newStartIndex, int newLength) => new(Source, newStartIndex, newLength);
public StringSegment Relocate(int newStartIndex, int newLength) =>
new(Source, newStartIndex, newLength);
public StringSegment Relocate(Capture capture) => Relocate(capture.Index, capture.Length);
public override string ToString() => Source.Substring(StartIndex, Length);
}
}

2
DiscordChatExporter.Core/Markdown/TextNode.cs
View File

@@ -1,3 +1,3 @@
namespace DiscordChatExporter.Core.Markdown;
internal record TextNode(string Text) : MarkdownNode;
internal record TextNode(string Text) : MarkdownNode;

2
DiscordChatExporter.Core/Markdown/TimestampNode.cs
View File

@@ -6,4 +6,4 @@ namespace DiscordChatExporter.Core.Markdown;
internal record TimestampNode(DateTimeOffset? Instant, string? Format) : MarkdownNode
{
public static TimestampNode Invalid { get; } = new(null, null);
}
}