Add command line interface and change solution structure (#26)

This commit is contained in:
Alexey Golub
2018-01-12 20:28:36 +01:00
committed by GitHub
parent 7da82f9ef4
commit 8515efe11b
73 changed files with 489 additions and 199 deletions

View File

@@ -0,0 +1,321 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Models;
using Newtonsoft.Json.Linq;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
{
public partial class DataService : IDataService, IDisposable
{
private const string ApiRoot = "https://discordapp.com/api/v6";
private readonly HttpClient _httpClient = new HttpClient();
private readonly Dictionary<string, Role> _roleCache = new Dictionary<string, Role>();
private readonly Dictionary<string, Channel> _channelCache = new Dictionary<string, Channel>();
private User ParseUser(JToken token)
{
var id = token["id"].Value<string>();
var discriminator = token["discriminator"].Value<int>();
var name = token["username"].Value<string>();
var avatarHash = token["avatar"].Value<string>();
return new User(id, discriminator, name, avatarHash);
}
private Role ParseRole(JToken token)
{
var id = token["id"].Value<string>();
var name = token["name"].Value<string>();
return new Role(id, name);
}
private Guild ParseGuild(JToken token)
{
var id = token["id"].Value<string>();
var name = token["name"].Value<string>();
var iconHash = token["icon"].Value<string>();
var roles = token["roles"].Select(ParseRole).ToArray();
return new Guild(id, name, iconHash, roles);
}
private Channel ParseChannel(JToken token)
{
// Get basic data
var id = token["id"].Value<string>();
var guildId = token["guild_id"]?.Value<string>();
var type = (ChannelType) token["type"].Value<int>();
var topic = token["topic"]?.Value<string>();
// Extract name based on type
string name;
if (type.IsEither(ChannelType.DirectTextChat, ChannelType.DirectGroupTextChat))
{
guildId = Guild.DirectMessages.Id;
var recipients = token["recipients"].Select(ParseUser);
name = recipients.Select(r => r.Name).JoinToString(", ");
}
else
{
name = token["name"].Value<string>();
}
return new Channel(id, guildId, name, topic, type);
}
private Message ParseMessage(JToken token)
{
// Get basic data
var id = token["id"].Value<string>();
var channelId = token["channel_id"].Value<string>();
var timeStamp = token["timestamp"].Value<DateTime>();
var editedTimeStamp = token["edited_timestamp"]?.Value<DateTime?>();
var content = token["content"].Value<string>();
var type = (MessageType) token["type"].Value<int>();
// Workarounds for non-default types
if (type == MessageType.RecipientAdd)
content = "Added a recipient.";
else if (type == MessageType.RecipientRemove)
content = "Removed a recipient.";
else if (type == MessageType.Call)
content = "Started a call.";
else if (type == MessageType.ChannelNameChange)
content = "Changed the channel name.";
else if (type == MessageType.ChannelIconChange)
content = "Changed the channel icon.";
else if (type == MessageType.ChannelPinnedMessage)
content = "Pinned a message.";
else if (type == MessageType.GuildMemberJoin)
content = "Joined the server.";
// Get author
var author = ParseUser(token["author"]);
// Get attachment
var attachments = new List<Attachment>();
foreach (var attachmentJson in token["attachments"].EmptyIfNull())
{
var attachmentId = attachmentJson["id"].Value<string>();
var attachmentUrl = attachmentJson["url"].Value<string>();
var attachmentType = attachmentJson["width"] != null
? AttachmentType.Image
: AttachmentType.Other;
var attachmentFileName = attachmentJson["filename"].Value<string>();
var attachmentFileSize = attachmentJson["size"].Value<long>();
var attachment = new Attachment(
attachmentId, attachmentType, attachmentUrl,
attachmentFileName, attachmentFileSize);
attachments.Add(attachment);
}
// Get user mentions
var mentionedUsers = token["mentions"].Select(ParseUser).ToArray();
// Get role mentions
var mentionedRoles = token["mention_roles"]
.Values<string>()
.Select(i => _roleCache.GetOrDefault(i) ?? Role.CreateDeletedRole(id))
.ToArray();
// Get channel mentions
var mentionedChannels = Regex.Matches(content, "<#(\\d+)>")
.Cast<Match>()
.Select(m => m.Groups[1].Value)
.ExceptBlank()
.Select(i => _channelCache.GetOrDefault(i) ?? Channel.CreateDeletedChannel(id))
.ToArray();
return new Message(id, channelId, type, author, timeStamp, editedTimeStamp, content, attachments,
mentionedUsers, mentionedRoles, mentionedChannels);
}
private async Task<string> GetStringAsync(string url)
{
using (var response = await _httpClient.GetAsync(url))
{
// Check status code
// We throw our own exception here because default one doesn't have status code
if (!response.IsSuccessStatusCode)
throw new HttpErrorStatusCodeException(response.StatusCode);
// Get content
return await response.Content.ReadAsStringAsync();
}
}
public async Task<Guild> GetGuildAsync(string token, string guildId)
{
// Form request url
var url = $"{ApiRoot}/guilds/{guildId}?token={token}";
// Get response
var content = await GetStringAsync(url);
// Parse
var guild = ParseGuild(JToken.Parse(content));
// Add roles to cache
foreach (var role in guild.Roles)
_roleCache[role.Id] = role;
return guild;
}
public async Task<Channel> GetChannelAsync(string token, string channelId)
{
// Form request url
var url = $"{ApiRoot}/channels/{channelId}?token={token}";
// Get response
var content = await GetStringAsync(url);
// Parse
var channel = ParseChannel(JToken.Parse(content));
// Add channel to cache
_channelCache[channel.Id] = channel;
return channel;
}
public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(string token, string guildId)
{
// Form request url
var url = $"{ApiRoot}/guilds/{guildId}/channels?token={token}";
// Get response
var content = await GetStringAsync(url);
// Parse
var channels = JArray.Parse(content).Select(ParseChannel).ToArray();
// Add channels to cache
foreach (var channel in channels)
_channelCache[channel.Id] = channel;
return channels;
}
public async Task<IReadOnlyList<Guild>> GetUserGuildsAsync(string token)
{
// Form request url
var url = $"{ApiRoot}/users/@me/guilds?token={token}&limit=100";
// Get response
var content = await GetStringAsync(url);
// Parse IDs
var guildIds = JArray.Parse(content).Select(t => t["id"].Value<string>());
// Get full guild infos
var guilds = new List<Guild>();
foreach (var guildId in guildIds)
{
var guild = await GetGuildAsync(token, guildId);
guilds.Add(guild);
}
return guilds;
}
public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(string token)
{
// Form request url
var url = $"{ApiRoot}/users/@me/channels?token={token}";
// Get response
var content = await GetStringAsync(url);
// Parse
var channels = JArray.Parse(content).Select(ParseChannel).ToArray();
return channels;
}
public async Task<IReadOnlyList<Message>> GetChannelMessagesAsync(string token, string channelId,
DateTime? from, DateTime? to)
{
var result = new List<Message>();
// We are going backwards from last message to first
// collecting everything between them in batches
var beforeId = to != null ? DateTimeToSnowflake(to.Value) : null;
while (true)
{
// Form request url
var url = $"{ApiRoot}/channels/{channelId}/messages?token={token}&limit=100";
if (beforeId.IsNotBlank())
url += $"&before={beforeId}";
// Get response
var content = await GetStringAsync(url);
// Parse
var messages = JArray.Parse(content).Select(ParseMessage);
// Add messages to list
string currentMessageId = null;
foreach (var message in messages)
{
// Break when the message is older than from date
if (from != null && message.TimeStamp < from)
{
currentMessageId = null;
break;
}
// Add message
result.Add(message);
currentMessageId = message.Id;
}
// If no messages - break
if (currentMessageId == null)
break;
// Otherwise offset the next request
beforeId = currentMessageId;
}
// Messages appear newest first, we need to reverse
result.Reverse();
return result;
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
_httpClient.Dispose();
}
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
}
public partial class DataService
{
private static string DateTimeToSnowflake(DateTime dateTime)
{
const long epoch = 62135596800000;
var unixTime = dateTime.ToUniversalTime().Ticks / TimeSpan.TicksPerMillisecond - epoch;
var value = ((ulong) unixTime - 1420070400000UL) << 22;
return value.ToString();
}
}
}

View File

@@ -0,0 +1,322 @@
using System;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Internal;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
{
public partial class ExportService : IExportService
{
private readonly ISettingsService _settingsService;
public ExportService(ISettingsService settingsService)
{
_settingsService = settingsService;
}
private async Task ExportAsTextAsync(string filePath, ChannelChatLog log)
{
using (var writer = new StreamWriter(filePath, false, Encoding.UTF8, 128 * 1024))
{
// Generation info
await writer.WriteLineAsync("https://github.com/Tyrrrz/DiscordChatExporter");
await writer.WriteLineAsync();
// Guild and channel info
await writer.WriteLineAsync('='.Repeat(48));
await writer.WriteLineAsync($"Guild: {log.Guild.Name}");
await writer.WriteLineAsync($"Channel: {log.Channel.Name}");
await writer.WriteLineAsync($"Topic: {log.Channel.Topic}");
await writer.WriteLineAsync($"Messages: {log.TotalMessageCount:N0}");
await writer.WriteLineAsync('='.Repeat(48));
await writer.WriteLineAsync();
// Chat log
foreach (var group in log.MessageGroups)
{
var timeStampFormatted = group.TimeStamp.ToString(_settingsService.DateFormat);
await writer.WriteLineAsync($"{group.Author.FullyQualifiedName} [{timeStampFormatted}]");
// Messages
foreach (var message in group.Messages)
{
// Content
if (message.Content.IsNotBlank())
{
var contentFormatted = FormatMessageContentText(message);
await writer.WriteLineAsync(contentFormatted);
}
// Attachments
foreach (var attachment in message.Attachments)
{
await writer.WriteLineAsync(attachment.Url);
}
}
await writer.WriteLineAsync();
}
}
}
private async Task ExportAsHtmlAsync(string filePath, ChannelChatLog log, string css)
{
using (var writer = new StreamWriter(filePath, false, Encoding.UTF8, 128 * 1024))
{
// Generation info
await writer.WriteLineAsync("<!-- https://github.com/Tyrrrz/DiscordChatExporter -->");
// Html start
await writer.WriteLineAsync("<!DOCTYPE html>");
await writer.WriteLineAsync("<html lang=\"en\">");
// HEAD
await writer.WriteLineAsync("<head>");
await writer.WriteLineAsync($"<title>{log.Guild.Name} - {log.Channel.Name}</title>");
await writer.WriteLineAsync("<meta charset=\"utf-8\" />");
await writer.WriteLineAsync("<meta name=\"viewport\" content=\"width=device-width\" />");
await writer.WriteLineAsync($"<style>{css}</style>");
await writer.WriteLineAsync("</head>");
// Body start
await writer.WriteLineAsync("<body>");
// Guild and channel info
await writer.WriteLineAsync("<div id=\"info\">");
await writer.WriteLineAsync("<div class=\"info-left\">");
await writer.WriteLineAsync($"<img class=\"guild-icon\" src=\"{log.Guild.IconUrl}\" />");
await writer.WriteLineAsync("</div>"); // info-left
await writer.WriteLineAsync("<div class=\"info-right\">");
await writer.WriteLineAsync($"<div class=\"guild-name\">{log.Guild.Name}</div>");
await writer.WriteLineAsync($"<div class=\"channel-name\">{log.Channel.Name}</div>");
await writer.WriteLineAsync($"<div class=\"channel-topic\">{log.Channel.Topic}</div>");
await writer.WriteLineAsync(
$"<div class=\"channel-messagecount\">{log.TotalMessageCount:N0} messages</div>");
await writer.WriteLineAsync("</div>"); // info-right
await writer.WriteLineAsync("</div>"); // info
// Chat log
await writer.WriteLineAsync("<div id=\"log\">");
foreach (var group in log.MessageGroups)
{
await writer.WriteLineAsync("<div class=\"msg\">");
await writer.WriteLineAsync("<div class=\"msg-left\">");
await writer.WriteLineAsync($"<img class=\"msg-avatar\" src=\"{group.Author.AvatarUrl}\" />");
await writer.WriteLineAsync("</div>");
await writer.WriteLineAsync("<div class=\"msg-right\">");
await writer.WriteAsync(
$"<span class=\"msg-user\" title=\"{HtmlEncode(group.Author.FullyQualifiedName)}\">");
await writer.WriteAsync(HtmlEncode(group.Author.Name));
await writer.WriteLineAsync("</span>");
var timeStampFormatted = HtmlEncode(group.TimeStamp.ToString(_settingsService.DateFormat));
await writer.WriteLineAsync($"<span class=\"msg-date\">{timeStampFormatted}</span>");
// Messages
foreach (var message in group.Messages)
{
// Content
if (message.Content.IsNotBlank())
{
await writer.WriteLineAsync("<div class=\"msg-content\">");
var contentFormatted = FormatMessageContentHtml(message);
await writer.WriteAsync(contentFormatted);
// Edited timestamp
if (message.EditedTimeStamp != null)
{
var editedTimeStampFormatted =
HtmlEncode(message.EditedTimeStamp.Value.ToString(_settingsService.DateFormat));
await writer.WriteAsync(
$"<span class=\"msg-edited\" title=\"{editedTimeStampFormatted}\">(edited)</span>");
}
await writer.WriteLineAsync("</div>"); // msg-content
}
// Attachments
foreach (var attachment in message.Attachments)
{
if (attachment.Type == AttachmentType.Image)
{
await writer.WriteLineAsync("<div class=\"msg-attachment\">");
await writer.WriteLineAsync($"<a href=\"{attachment.Url}\">");
await writer.WriteLineAsync(
$"<img class=\"msg-attachment\" src=\"{attachment.Url}\" />");
await writer.WriteLineAsync("</a>");
await writer.WriteLineAsync("</div>");
}
else
{
await writer.WriteLineAsync("<div class=\"msg-attachment\">");
await writer.WriteLineAsync($"<a href=\"{attachment.Url}\">");
var fileSizeFormatted = FormatFileSize(attachment.FileSize);
await writer.WriteLineAsync($"Attachment: {attachment.FileName} ({fileSizeFormatted})");
await writer.WriteLineAsync("</a>");
await writer.WriteLineAsync("</div>");
}
}
}
await writer.WriteLineAsync("</div>"); // msg-right
await writer.WriteLineAsync("</div>"); // msg
}
await writer.WriteLineAsync("</div>"); // log
await writer.WriteLineAsync("</body>");
await writer.WriteLineAsync("</html>");
}
}
public Task ExportAsync(ExportFormat format, string filePath, ChannelChatLog log)
{
if (format == ExportFormat.PlainText)
{
return ExportAsTextAsync(filePath, log);
}
if (format == ExportFormat.HtmlDark)
{
var css = AssemblyHelper.GetResourceString("DiscordChatExporter.Core.Resources.ExportService.DarkTheme.css");
return ExportAsHtmlAsync(filePath, log, css);
}
if (format == ExportFormat.HtmlLight)
{
var css = AssemblyHelper.GetResourceString("DiscordChatExporter.Core.Resources.ExportService.LightTheme.css");
return ExportAsHtmlAsync(filePath, log, css);
}
throw new ArgumentOutOfRangeException(nameof(format));
}
}
public partial class ExportService
{
private static string HtmlEncode(string str)
{
return WebUtility.HtmlEncode(str);
}
private static string HtmlEncode(object obj)
{
return WebUtility.HtmlEncode(obj.ToString());
}
private static string FormatFileSize(long fileSize)
{
string[] units = {"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"};
double size = fileSize;
var unit = 0;
while (size >= 1024)
{
size /= 1024;
++unit;
}
return $"{size:0.#} {units[unit]}";
}
public static string FormatMessageContentText(Message message)
{
var content = message.Content;
// New lines
content = content.Replace("\n", Environment.NewLine);
// User mentions (<@id> and <@!id>)
foreach (var mentionedUser in message.MentionedUsers)
content = Regex.Replace(content, $"<@!?{mentionedUser.Id}>", $"@{mentionedUser}");
// Role mentions (<@&id>)
foreach (var mentionedRole in message.MentionedRoles)
content = content.Replace($"<@&{mentionedRole.Id}>", $"@{mentionedRole.Name}");
// Channel mentions (<#id>)
foreach (var mentionedChannel in message.MentionedChannels)
content = content.Replace($"<#{mentionedChannel.Id}>", $"#{mentionedChannel.Name}");
// Custom emojis (<:name:id>)
content = Regex.Replace(content, "<(:.*?:)\\d*>", "$1");
return content;
}
private static string FormatMessageContentHtml(Message message)
{
var content = message.Content;
// Encode HTML
content = HtmlEncode(content);
// Pre multiline (```text```)
content = Regex.Replace(content, "```+(?:[^`]*?\\n)?([^`]+)\\n?```+", "<div class=\"pre\">$1</div>");
// Pre inline (`text`)
content = Regex.Replace(content, "`([^`]+)`", "<span class=\"pre\">$1</span>");
// Bold (**text**)
content = Regex.Replace(content, "\\*\\*([^\\*]*?)\\*\\*", "<b>$1</b>");
// Italic (*text*)
content = Regex.Replace(content, "\\*([^\\*]*?)\\*", "<i>$1</i>");
// Underline (__text__)
content = Regex.Replace(content, "__([^_]*?)__", "<u>$1</u>");
// Italic (_text_)
content = Regex.Replace(content, "_([^_]*?)_", "<i>$1</i>");
// Strike through (~~text~~)
content = Regex.Replace(content, "~~([^~]*?)~~", "<s>$1</s>");
// New lines
content = content.Replace("\n", "<br />");
// URL links
content = Regex.Replace(content, "((https?|ftp)://[^\\s/$.?#].[^\\s<>]*)", "<a href=\"$1\">$1</a>");
// Meta mentions (@everyone)
content = content.Replace("@everyone", "<span class=\"mention\">@everyone</span>");
// Meta mentions (@here)
content = content.Replace("@here", "<span class=\"mention\">@here</span>");
// User mentions (<@id> and <@!id>)
foreach (var mentionedUser in message.MentionedUsers)
{
content = Regex.Replace(content, $"&lt;@!?{mentionedUser.Id}&gt;",
$"<span class=\"mention\" title=\"{HtmlEncode(mentionedUser)}\">" +
$"@{HtmlEncode(mentionedUser.Name)}" +
"</span>");
}
// Role mentions (<@&id>)
foreach (var mentionedRole in message.MentionedRoles)
{
content = content.Replace($"&lt;@&amp;{mentionedRole.Id}&gt;",
"<span class=\"mention\">" +
$"@{HtmlEncode(mentionedRole.Name)}" +
"</span>");
}
// Channel mentions (<#id>)
foreach (var mentionedChannel in message.MentionedChannels)
{
content = content.Replace($"&lt;#{mentionedChannel.Id}&gt;",
"<span class=\"mention\">" +
$"#{HtmlEncode(mentionedChannel.Name)}" +
"</span>");
}
// Custom emojis (<:name:id>)
content = Regex.Replace(content, "&lt;(:.*?:)(\\d*)&gt;",
"<img class=\"emoji\" title=\"$1\" src=\"https://cdn.discordapp.com/emojis/$2.png\" />");
return content;
}
}
}

View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services
{
public interface IDataService
{
Task<Guild> GetGuildAsync(string token, string guildId);
Task<Channel> GetChannelAsync(string token, string channelId);
Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(string token, string guildId);
Task<IReadOnlyList<Guild>> GetUserGuildsAsync(string token);
Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(string token);
Task<IReadOnlyList<Message>> GetChannelMessagesAsync(string token, string channelId,
DateTime? from, DateTime? to);
}
}

View File

@@ -0,0 +1,10 @@
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services
{
public interface IExportService
{
Task ExportAsync(ExportFormat format, string filePath, ChannelChatLog log);
}
}

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services
{
public interface IMessageGroupService
{
IReadOnlyList<MessageGroup> GroupMessages(IReadOnlyList<Message> messages);
}
}

View File

@@ -0,0 +1,16 @@
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services
{
public interface ISettingsService
{
string DateFormat { get; set; }
int MessageGroupLimit { get; set; }
string LastToken { get; set; }
ExportFormat LastExportFormat { get; set; }
void Load();
void Save();
}
}

View File

@@ -0,0 +1,59 @@
using System.Collections.Generic;
using System.Linq;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services
{
public class MessageGroupService : IMessageGroupService
{
private readonly ISettingsService _settingsService;
public MessageGroupService(ISettingsService settingsService)
{
_settingsService = settingsService;
}
public IReadOnlyList<MessageGroup> GroupMessages(IReadOnlyList<Message> messages)
{
var result = new List<MessageGroup>();
// Group adjacent messages by timestamp and author
var groupBuffer = new List<Message>();
foreach (var message in messages)
{
var groupFirst = groupBuffer.FirstOrDefault();
// Group break condition
var breakCondition =
groupFirst != null &&
(
message.Author.Id != groupFirst.Author.Id ||
(message.TimeStamp - groupFirst.TimeStamp).TotalHours > 1 ||
message.TimeStamp.Hour != groupFirst.TimeStamp.Hour ||
groupBuffer.Count >= _settingsService.MessageGroupLimit
);
// If condition is true - flush buffer
if (breakCondition)
{
var group = new MessageGroup(groupFirst.Author, groupFirst.TimeStamp, groupBuffer.ToArray());
result.Add(group);
groupBuffer.Clear();
}
// Add message to buffer
groupBuffer.Add(message);
}
// Add what's remaining in buffer
if (groupBuffer.Any())
{
var groupFirst = groupBuffer.First();
var group = new MessageGroup(groupFirst.Author, groupFirst.TimeStamp, groupBuffer.ToArray());
result.Add(group);
}
return result;
}
}
}

View File

@@ -0,0 +1,21 @@
using DiscordChatExporter.Core.Models;
using Tyrrrz.Settings;
namespace DiscordChatExporter.Core.Services
{
public class SettingsService : SettingsManager, ISettingsService
{
public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
public int MessageGroupLimit { get; set; } = 20;
public string LastToken { get; set; }
public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;
public SettingsService()
{
Configuration.StorageSpace = StorageSpace.Instance;
Configuration.SubDirectoryPath = "";
Configuration.FileName = "Settings.dat";
}
}
}