This commit is contained in:
Alexey Golub
2020-04-22 20:44:14 +03:00
parent 6a430cdc76
commit dc79813ad7
13 changed files with 227 additions and 263 deletions

View File

@@ -31,8 +31,7 @@ namespace DiscordChatExporter.Domain.Exporting
var mentionableChannels = await _discord.GetGuildChannelsAsync(guild.Id);
var mentionableRoles = guild.Roles;
var context = new RenderContext
(
var context = new RenderContext(
guild, channel, after, before, dateFormat,
mentionableUsers, mentionableChannels, mentionableRoles
);

View File

@@ -9,7 +9,7 @@ using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
internal class CsvMessageWriter : MessageWriterBase
internal partial class CsvMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
@@ -19,35 +19,26 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer = new StreamWriter(stream);
}
private string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
private string FormatMessage(Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(message.Content))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
}
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
public override async Task WritePreambleAsync() =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async Task WriteMessageAsync(Message message) =>
await _writer.WriteLineAsync(FormatMessage(message));
public override async Task WriteMessageAsync(Message message)
{
var buffer = new StringBuilder();
buffer
.Append(CsvEncode(message.Author.Id)).Append(',')
.Append(CsvEncode(message.Author.FullName)).Append(',')
.Append(CsvEncode(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(CsvEncode(FormatMarkdown(message.Content))).Append(',')
.Append(CsvEncode(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(CsvEncode(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
await _writer.WriteLineAsync(buffer.ToString());
}
public override async ValueTask DisposeAsync()
{
@@ -55,4 +46,13 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
await base.DisposeAsync();
}
}
internal partial class CsvMessageWriter
{
private static string CsvEncode(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
}
}

View File

@@ -1,17 +1,12 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions;
@@ -79,7 +74,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
new Func<DateTimeOffset, string>(d => d.ToLocalString(Context.DateFormat)));
scriptObject.Import("FormatMarkdown",
new Func<string, string>(FormatMarkdown));
new Func<string?, string>(FormatMarkdown));
scriptObject.Import("GetUserColor", new Func<Guild, User, string>(Guild.GetUserColor));
@@ -94,8 +89,8 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
return templateContext;
}
private string FormatMarkdown(string markdown) =>
HtmlMarkdownVisitor.Format(Context, markdown);
private string FormatMarkdown(string? markdown) =>
HtmlMarkdownVisitor.Format(Context, markdown ?? "");
private async Task RenderCurrentMessageGroupAsync()
{
@@ -180,7 +175,5 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
}
}

View File

@@ -22,6 +22,124 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
});
}
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private void WriteAttachment(Attachment attachment)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("url", attachment.Url);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
}
private void WriteEmbedAuthor(EmbedAuthor embedAuthor)
{
_writer.WriteStartObject("author");
_writer.WriteString("name", embedAuthor.Name);
_writer.WriteString("url", embedAuthor.Url);
_writer.WriteString("iconUrl", embedAuthor.IconUrl);
_writer.WriteEndObject();
}
private void WriteEmbedThumbnail(EmbedImage embedThumbnail)
{
_writer.WriteStartObject("thumbnail");
_writer.WriteString("url", embedThumbnail.Url);
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject();
}
private void WriteEmbedImage(EmbedImage embedImage)
{
_writer.WriteStartObject("image");
_writer.WriteString("url", embedImage.Url);
_writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height);
_writer.WriteEndObject();
}
private void WriteEmbedFooter(EmbedFooter embedFooter)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embedFooter.Text);
_writer.WriteString("iconUrl", embedFooter.IconUrl);
_writer.WriteEndObject();
}
private void WriteEmbedField(EmbedField embedField)
{
_writer.WriteStartObject();
_writer.WriteString("name", embedField.Name);
_writer.WriteString("value", embedField.Value);
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
}
private void WriteEmbed(Embed embed)
{
_writer.WriteStartObject();
_writer.WriteString("title", FormatMarkdown(embed.Title));
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", FormatMarkdown(embed.Description));
if (embed.Author != null)
WriteEmbedAuthor(embed.Author);
if (embed.Thumbnail != null)
WriteEmbedThumbnail(embed.Thumbnail);
if (embed.Image != null)
WriteEmbedImage(embed.Image);
if (embed.Footer != null)
WriteEmbedFooter(embed.Footer);
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
WriteEmbedField(field);
_writer.WriteEndArray();
_writer.WriteEndObject();
}
private void WriteReaction(Reaction reaction)
{
_writer.WriteStartObject();
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", reaction.Emoji.ImageUrl);
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
}
public override async Task WritePreambleAsync()
{
// Root object (start)
@@ -66,8 +184,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteBoolean("isPinned", message.IsPinned);
// Content
var content = PlainTextMarkdownVisitor.Format(Context, message.Content);
_writer.WriteString("content", content);
_writer.WriteString("content", FormatMarkdown(message.Content));
// Author
_writer.WriteStartObject("author");
@@ -82,16 +199,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("attachments");
foreach (var attachment in message.Attachments)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("url", attachment.Url);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
}
WriteAttachment(attachment);
_writer.WriteEndArray();
@@ -99,71 +207,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("embeds");
foreach (var embed in message.Embeds)
{
_writer.WriteStartObject();
_writer.WriteString("title", embed.Title);
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", embed.Description);
// Author
if (embed.Author != null)
{
_writer.WriteStartObject("author");
_writer.WriteString("name", embed.Author.Name);
_writer.WriteString("url", embed.Author.Url);
_writer.WriteString("iconUrl", embed.Author.IconUrl);
_writer.WriteEndObject();
}
// Thumbnail
if (embed.Thumbnail != null)
{
_writer.WriteStartObject("thumbnail");
_writer.WriteString("url", embed.Thumbnail.Url);
_writer.WriteNumber("width", embed.Thumbnail.Width);
_writer.WriteNumber("height", embed.Thumbnail.Height);
_writer.WriteEndObject();
}
// Image
if (embed.Image != null)
{
_writer.WriteStartObject("image");
_writer.WriteString("url", embed.Image.Url);
_writer.WriteNumber("width", embed.Image.Width);
_writer.WriteNumber("height", embed.Image.Height);
_writer.WriteEndObject();
}
// Footer
if (embed.Footer != null)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embed.Footer.Text);
_writer.WriteString("iconUrl", embed.Footer.IconUrl);
_writer.WriteEndObject();
}
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
{
_writer.WriteStartObject();
_writer.WriteString("name", field.Name);
_writer.WriteString("value", field.Value);
_writer.WriteBoolean("isInline", field.IsInline);
_writer.WriteEndObject();
}
_writer.WriteEndArray();
_writer.WriteEndObject();
}
WriteEmbed(embed);
_writer.WriteEndArray();
@@ -171,31 +215,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("reactions");
foreach (var reaction in message.Reactions)
{
_writer.WriteStartObject();
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", reaction.Emoji.ImageUrl);
_writer.WriteEndObject();
// Count
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
}
WriteReaction(reaction);
_writer.WriteEndArray();
_writer.WriteEndObject();
_messageCount++;
// Flush every 100 messages
if (_messageCount % 100 == 0)
if (_messageCount++ % 100 == 0)
await _writer.FlushAsync();
}
@@ -204,7 +231,6 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Message array (end)
_writer.WriteEndArray();
// Message count
_writer.WriteNumber("messageCount", _messageCount);
// Root object (end)

View File

@@ -115,7 +115,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
: "";
_buffer
.Append($"<span class=\"mention\" style=\"{style}>\"")
.Append($"<span class=\"mention\" style=\"{style}\">")
.Append("@").Append(HtmlEncode(role.Name))
.Append("</span>");
}

View File

@@ -52,9 +52,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
public override MarkdownNode VisitEmoji(EmojiNode emoji)
{
_buffer.Append(emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name);
_buffer.Append(
emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name
);
return base.VisitEmoji(emoji);
}

View File

@@ -22,41 +22,8 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer = new StreamWriter(stream);
}
private string FormatPreamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatPostamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Exported {_messageCount:N0} message(s)");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private string FormatMessageHeader(Message message)
{
@@ -70,21 +37,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
private string FormatMessageContent(Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(message.Content);
}
private string FormatAttachments(IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
@@ -109,49 +66,25 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
buffer
.AppendLine("{Embed}")
.AppendLineIfNotNullOrWhiteSpace(embed.Author?.Name)
.AppendLineIfNotNullOrWhiteSpace(embed.Url)
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Title))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Description));
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
buffer
.AppendLineIfNotNullOrWhiteSpace(field.Name)
.AppendLineIfNotNullOrWhiteSpace(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
buffer
.AppendLineIfNotNullOrWhiteSpace(embed.Thumbnail?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Image?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Footer?.Text)
.AppendLine();
}
return buffer.ToString();
@@ -187,18 +120,35 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
buffer
.AppendLine(FormatMessageHeader(message))
.AppendLineIfNotEmpty(FormatMessageContent(message))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(message.Content))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(message.Reactions));
.AppendLineIfNotNullOrWhiteSpace(FormatAttachments(message.Attachments))
.AppendLineIfNotNullOrWhiteSpace(FormatEmbeds(message.Embeds))
.AppendLineIfNotNullOrWhiteSpace(FormatReactions(message.Reactions));
return buffer.Trim().ToString();
}
public override async Task WritePreambleAsync()
{
await _writer.WriteLineAsync(FormatPreamble());
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
}
public override async Task WriteMessageAsync(Message message)
@@ -211,8 +161,15 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async Task WritePostambleAsync()
{
await _writer.WriteLineAsync();
await _writer.WriteLineAsync(FormatPostamble());
var buffer = new StringBuilder();
buffer
.Append('=', 62).AppendLine()
.AppendLine($"Exported {_messageCount:N0} message(s)")
.Append('=', 62).AppendLine()
.AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
}
public override async ValueTask DisposeAsync()