Streaming exporter

Fixes #125
Closes #177
This commit is contained in:
Alexey Golub
2019-12-07 18:43:24 +02:00
parent fc38afe6a0
commit 2a223599f9
44 changed files with 1132 additions and 1098 deletions

View File

@@ -0,0 +1,39 @@
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class CsvRenderingLogic
{
// Header is always the same
public static string FormatHeader(RenderContext context) => "AuthorID,Author,Date,Content,Attachments,Reactions";
private static string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
PlainTextRenderingLogic.FormatMarkdown(context, markdown);
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(FormatDate(message.Timestamp, context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(context, message.Content ?? ""))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
}
}
}

View File

@@ -0,0 +1,160 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering.Logic
{
internal static class HtmlRenderingLogic
{
public static bool CanBeGrouped(Message message1, Message message2)
{
if (message1.Author.Id != message2.Author.Id)
return false;
if ((message2.Timestamp - message1.Timestamp).Duration().TotalMinutes > 7)
return false;
return true;
}
private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
private static string FormatMarkdownNode(RenderContext context, Node node, bool isJumbo)
{
// Text node
if (node is TextNode textNode)
{
// Return HTML-encoded text
return HtmlEncode(textNode.Text);
}
// Formatted node
if (node is FormattedNode formattedNode)
{
// Recursively get inner html
var innerHtml = FormatMarkdownNodes(context, formattedNode.Children, false);
// Bold
if (formattedNode.Formatting == TextFormatting.Bold)
return $"<strong>{innerHtml}</strong>";
// Italic
if (formattedNode.Formatting == TextFormatting.Italic)
return $"<em>{innerHtml}</em>";
// Underline
if (formattedNode.Formatting == TextFormatting.Underline)
return $"<u>{innerHtml}</u>";
// Strikethrough
if (formattedNode.Formatting == TextFormatting.Strikethrough)
return $"<s>{innerHtml}</s>";
// Spoiler
if (formattedNode.Formatting == TextFormatting.Spoiler)
return $"<span class=\"spoiler\">{innerHtml}</span>";
// Quote
if (formattedNode.Formatting == TextFormatting.Quote)
return $"<div class=\"quote\">{innerHtml}</div>";
}
// Inline code block node
if (node is InlineCodeBlockNode inlineCodeBlockNode)
{
return $"<span class=\"pre pre--inline\">{HtmlEncode(inlineCodeBlockNode.Code)}</span>";
}
// Multi-line code block node
if (node is MultiLineCodeBlockNode multilineCodeBlockNode)
{
// Set CSS class for syntax highlighting
var highlightCssClass = !string.IsNullOrWhiteSpace(multilineCodeBlockNode.Language)
? $"language-{multilineCodeBlockNode.Language}"
: "nohighlight";
return $"<div class=\"pre pre--multiline {highlightCssClass}\">{HtmlEncode(multilineCodeBlockNode.Code)}</div>";
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return $"<span class=\"mention\">@{HtmlEncode(mentionNode.Id)}</span>";
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
return $"<span class=\"mention\" title=\"{HtmlEncode(user.FullName)}\">@{HtmlEncode(user.Name)}</span>";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"<span class=\"mention\">#{HtmlEncode(channel.Name)}</span>";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
return $"<span class=\"mention\">@{HtmlEncode(role.Name)}</span>";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
// Get emoji image URL
var emojiImageUrl = Emoji.GetImageUrl(emojiNode.Id, emojiNode.Name, emojiNode.IsAnimated);
// Make emoji large if it's jumbo
var jumboableCssClass = isJumbo ? "emoji--large" : null;
return $"<img class=\"emoji {jumboableCssClass}\" alt=\"{emojiNode.Name}\" title=\"{emojiNode.Name}\" src=\"{emojiImageUrl}\" />";
}
// Link node
if (node is LinkNode linkNode)
{
// Extract message ID if the link points to a Discord message
var linkedMessageId = Regex.Match(linkNode.Url, "^https?://discordapp.com/channels/.*?/(\\d+)/?$").Groups[1].Value;
return string.IsNullOrWhiteSpace(linkedMessageId)
? $"<a href=\"{Uri.EscapeUriString(linkNode.Url)}\">{HtmlEncode(linkNode.Title)}</a>"
: $"<a href=\"{Uri.EscapeUriString(linkNode.Url)}\" onclick=\"scrollToMessage(event, '{linkedMessageId}')\">{HtmlEncode(linkNode.Title)}</a>";
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
}
private static string FormatMarkdownNodes(RenderContext context, IReadOnlyList<Node> nodes, bool isTopLevel)
{
// Emojis are jumbo if all top-level nodes are emoji nodes or whitespace text nodes
var isJumbo = isTopLevel && nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
return nodes.Select(n => FormatMarkdownNode(context, n, isJumbo)).JoinToString("");
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
FormatMarkdownNodes(context, MarkdownParser.Parse(markdown), true);
}
}

View File

@@ -0,0 +1,235 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Internal;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class PlainTextRenderingLogic
{
public static string FormatPreamble(RenderContext context)
{
var buffer = new StringBuilder();
buffer.AppendLine('='.Repeat(62));
buffer.AppendLine($"Guild: {context.Guild.Name}");
buffer.AppendLine($"Channel: {context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(context.Channel.Topic))
buffer.AppendLine($"Topic: {context.Channel.Topic}");
if (context.After != null)
buffer.AppendLine($"After: {FormatDate(context.After.Value, context.DateFormat)}");
if (context.Before != null)
buffer.AppendLine($"Before: {FormatDate(context.Before.Value, context.DateFormat)}");
buffer.AppendLine('='.Repeat(62));
return buffer.ToString();
}
private static string FormatMarkdownNode(RenderContext context, Node node)
{
// Text node
if (node is TextNode textNode)
{
return textNode.Text;
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return $"@{mentionNode.Id}";
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
return $"@{user.Name}";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"#{channel.Name}";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
return $"@{role.Name}";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
return emojiNode.IsCustomEmoji ? $":{emojiNode.Name}:" : emojiNode.Name;
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
MarkdownParser.ParseMinimal(markdown).Select(n => FormatMarkdownNode(context, n)).JoinToString("");
public static string FormatMessageHeader(RenderContext context, Message message)
{
var buffer = new StringBuilder();
// Timestamp & author
buffer
.Append($"[{FormatDate(message.Timestamp, context.DateFormat)}]")
.Append(' ')
.Append($"{message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
public static string FormatMessageContent(RenderContext context, Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(context, message.Content);
}
public static string FormatAttachments(RenderContext context, IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
return "";
var buffer = new StringBuilder();
buffer
.AppendLine("{Attachments}")
.AppendJoin(Environment.NewLine, attachments.Select(a => a.Url))
.AppendLine();
return buffer.ToString();
}
public static string FormatEmbeds(RenderContext context, IReadOnlyList<Embed> embeds)
{
if (!embeds.Any())
return "";
var buffer = new StringBuilder();
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(context, embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(context, embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
}
return buffer.ToString();
}
public static string FormatReactions(RenderContext context, IReadOnlyList<Reaction> reactions)
{
if (!reactions.Any())
return "";
var buffer = new StringBuilder();
buffer.AppendLine("{Reactions}");
foreach (var reaction in reactions)
{
buffer.Append(reaction.Emoji.Name);
if (reaction.Count > 1)
buffer.Append($" ({reaction.Count})");
buffer.Append(" ");
}
buffer.AppendLine();
return buffer.ToString();
}
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.AppendLine(FormatMessageHeader(context, message))
.AppendLineIfNotEmpty(FormatMessageContent(context, message))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(context, message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(context, message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(context, message.Reactions));
return buffer.Trim().ToString();
}
}
}

View File

@@ -0,0 +1,11 @@
using System;
using System.Globalization;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class SharedRenderingLogic
{
public static string FormatDate(DateTimeOffset date, string dateFormat) =>
date.ToLocalTime().ToString(dateFormat, CultureInfo.InvariantCulture);
}
}