Add support for embeds (#46)

DiscordChatExporter.Core/Services/DataService.cs

@@ -8,6 +8,8 @@ using DiscordChatExporter.Core.Exceptions;
 using DiscordChatExporter.Core.Models;
 using Newtonsoft.Json.Linq;
 using Tyrrrz.Extensions;
+using System.Drawing;
+using System.Numerics;
 
 namespace DiscordChatExporter.Core.Services
 {
@@ -16,6 +18,7 @@ namespace DiscordChatExporter.Core.Services
         private const string ApiRoot = "https://discordapp.com/api/v6";
 
         private readonly HttpClient _httpClient = new HttpClient();
+        private readonly Dictionary<string, User> _userCache = new Dictionary<string, User>();
         private readonly Dictionary<string, Role> _roleCache = new Dictionary<string, Role>();
         private readonly Dictionary<string, Channel> _channelCache = new Dictionary<string, Channel>();
 
@@ -76,6 +79,113 @@ namespace DiscordChatExporter.Core.Services
             return new Channel(id, guildId, name, topic, type);
         }
 
+        private Embed ParseEmbed(JToken token)
+        {
+
+            // var embedFileSize = embedJson["size"].Value<long>();
+            var title = token["title"]?.Value<string>();
+            var type = token["type"]?.Value<string>();
+            var description = token["description"]?.Value<string>();
+            var url = token["url"]?.Value<string>();
+            var timestamp = token["timestamp"]?.Value<DateTime>();
+            var color = token["color"] != null
+                ? Color.FromArgb(token["color"].Value<int>())
+                : (Color?)null;
+
+            var footerNode = token["footer"];
+            var footer = footerNode != null
+                ? new EmbedFooter(
+                    footerNode["text"]?.Value<string>(),
+                    footerNode["icon_url"]?.Value<string>(),
+                    footerNode["proxy_icon_url"]?.Value<string>())
+                : null;
+
+            var imageNode = token["image"];
+            var image = imageNode != null
+                ? new EmbedImage(
+                    imageNode["url"]?.Value<string>(),
+                    imageNode["proxy_url"]?.Value<string>(),
+                    imageNode["height"]?.Value<int>(),
+                    imageNode["width"]?.Value<int>())
+                : null;
+
+            var thumbnailNode = token["thumbnail"];
+            var thumbnail = thumbnailNode != null
+                ? new EmbedImage(
+                    thumbnailNode["url"]?.Value<string>(),
+                    thumbnailNode["proxy_url"]?.Value<string>(),
+                    thumbnailNode["height"]?.Value<int>(),
+                    thumbnailNode["width"]?.Value<int>())
+                : null;
+
+            var videoNode = token["video"];
+            var video = videoNode != null
+                ? new EmbedVideo(
+                    videoNode["url"]?.Value<string>(),
+                    videoNode["height"]?.Value<int>(),
+                    videoNode["width"]?.Value<int>())
+                : null;
+
+            var providerNode = token["provider"];
+            var provider = providerNode != null
+                ? new EmbedProvider(
+                    providerNode["name"]?.Value<string>(),
+                    providerNode["url"]?.Value<string>())
+                : null;
+
+            var authorNode = token["author"];
+            var author = authorNode != null
+                ? new EmbedAuthor(
+                    authorNode["name"]?.Value<string>(),
+                    authorNode["url"]?.Value<string>(),
+                    authorNode["icon_url"]?.Value<string>(),
+                    authorNode["proxy_icon_url"]?.Value<string>())
+                : null;
+
+            var fields = new List<EmbedField>();
+            foreach (var fieldNode in token["fields"].EmptyIfNull())
+            {
+                fields.Add(new EmbedField(
+                    fieldNode["name"]?.Value<string>(),
+                    fieldNode["value"]?.Value<string>(),
+                    fieldNode["inline"]?.Value<bool>()));
+            }
+
+            var mentionableContent = description ?? "";
+            fields.ForEach(f => mentionableContent += f.Value);
+
+            // Get user mentions
+            var mentionedUsers = Regex.Matches(mentionableContent, "<@!?(\\d+)>")
+                .Cast<Match>()
+                .Select(m => m.Groups[1].Value)
+                .ExceptBlank()
+                .Select(i => _userCache.GetOrDefault(i) ?? User.CreateUnknownUser(i))
+                .ToList();
+
+            // Get role mentions
+            var mentionedRoles = Regex.Matches(mentionableContent, "<@&(\\d+)>")
+                .Cast<Match>()
+                .Select(m => m.Groups[1].Value)
+                .ExceptBlank()
+                .Select(i => _roleCache.GetOrDefault(i) ?? Role.CreateDeletedRole(i))
+                .ToList();
+
+            // Get channel mentions
+            var mentionedChannels = Regex.Matches(mentionableContent, "<#(\\d+)>")
+                .Cast<Match>()
+                .Select(m => m.Groups[1].Value)
+                .ExceptBlank()
+                .Select(i => _channelCache.GetOrDefault(i) ?? Channel.CreateDeletedChannel(i))
+                .ToList();
+
+            return new Embed(
+                title, type, description,
+                url, timestamp, color,
+                footer, image, thumbnail,
+                video, provider, author,
+                fields, mentionedUsers, mentionedRoles, mentionedChannels);
+        }
+
         private Message ParseMessage(JToken token)
         {
             // Get basic data
@@ -123,27 +233,64 @@ namespace DiscordChatExporter.Core.Services
                 attachments.Add(attachment);
             }
 
+            // Get embeds
+            var embeds = token["embeds"].EmptyIfNull().Select(ParseEmbed).ToArray();
+
             // Get user mentions
-            var mentionedUsers = token["mentions"].Select(ParseUser).ToArray();
+            var mentionedUsers = token["mentions"].Select(ParseUser).ToList();
 
             // Get role mentions
             var mentionedRoles = token["mention_roles"]
                 .Values<string>()
-                .Select(i => _roleCache.GetOrDefault(i) ?? Role.CreateDeletedRole(id))
-                .ToArray();
+                .Select(i => _roleCache.GetOrDefault(i) ?? Role.CreateDeletedRole(i))
+                .ToList();
 
             // Get channel mentions
             var mentionedChannels = Regex.Matches(content, "<#(\\d+)>")
                 .Cast<Match>()
                 .Select(m => m.Groups[1].Value)
                 .ExceptBlank()
-                .Select(i => _channelCache.GetOrDefault(i) ?? Channel.CreateDeletedChannel(id))
-                .ToArray();
+                .Select(i => _channelCache.GetOrDefault(i) ?? Channel.CreateDeletedChannel(i))
+                .ToList();
 
-            return new Message(id, channelId, type, author, timeStamp, editedTimeStamp, content, attachments,
+            return new Message(id, channelId, type, author, timeStamp, editedTimeStamp, content, attachments, embeds,
                 mentionedUsers, mentionedRoles, mentionedChannels);
         }
 
+        /// <summary>
+        /// Attempts to query for users, channels, and roles if they havent been found yet, and set them in the mentionable
+        /// </summary>
+        private async Task FillMentionable(string token, string guildId, IMentionable mentionable)
+        {
+            for (int i = 0; i < mentionable.MentionedUsers.Count; i++)
+            {
+                var user = mentionable.MentionedUsers[i];
+                if (user.Name == "Unknown" && user.Discriminator == 0)
+                {
+                    try
+                    {
+                        mentionable.MentionedUsers[i] = _userCache.GetOrDefault(user.Id) ?? (await GetMemberAsync(token, guildId, user.Id));
+                    }
+                    catch (HttpErrorStatusCodeException e) { } // This likely means the user doesnt exist any more, so ignore
+                }
+            }
+
+            for (int i = 0; i < mentionable.MentionedChannels.Count; i++)
+            {
+                var channel = mentionable.MentionedChannels[i];
+                if (channel.Name == "deleted-channel" && channel.GuildId == null)
+                {
+                    try
+                    {
+                        mentionable.MentionedChannels[i] = _channelCache.GetOrDefault(channel.Id) ?? (await GetChannelAsync(token, channel.Id));
+                    }
+                    catch (HttpErrorStatusCodeException e) { } // This likely means the user doesnt exist any more, so ignore
+                }
+            }
+
+            // Roles are already gotten via GetGuildRolesAsync at the start
+        } 
+
         private async Task<string> GetStringAsync(string url)
         {
             using (var response = await _httpClient.GetAsync(url))
@@ -193,6 +340,23 @@ namespace DiscordChatExporter.Core.Services
             return channel;
         }
 
+        public async Task<User> GetMemberAsync(string token, string guildId, string memberId)
+        {
+            // Form request url
+            var url = $"{ApiRoot}/guilds/{guildId}/members/{memberId}?token={token}";
+
+            // Get response
+            var content = await GetStringAsync(url);
+
+            // Parse
+            var user = ParseUser(JToken.Parse(content)["user"]);
+
+            // Add user to cache
+            _userCache[user.Id] = user;
+
+            return user;
+        }
+
         public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(string token, string guildId)
         {
             // Form request url
@@ -211,6 +375,25 @@ namespace DiscordChatExporter.Core.Services
             return channels;
         }
 
+
+        public async Task<IReadOnlyList<Role>> GetGuildRolesAsync(string token, string guildId)
+        {
+            // Form request url
+            var url = $"{ApiRoot}/guilds/{guildId}/roles?token={token}";
+
+            // Get response
+            var content = await GetStringAsync(url);
+
+            // Parse
+            var roles = JArray.Parse(content).Select(ParseRole).ToArray();
+
+            // Add roles to cache
+            foreach (var role in roles)
+                _roleCache[role.Id] = role;
+
+            return roles;
+        }
+
         public async Task<IReadOnlyList<Guild>> GetUserGuildsAsync(string token)
         {
             // Form request url
@@ -247,9 +430,60 @@ namespace DiscordChatExporter.Core.Services
             return channels;
         }
 
+        public async Task<IReadOnlyList<User>> GetGuildMembersAsync(string token, string guildId)
+        {
+            var result = new List<User>();
+            
+            var afterId = "";
+            while (true)
+            {
+                // Form request url
+                var url = $"{ApiRoot}/guilds/{guildId}/members?token={token}&limit=1000";
+                if (afterId.IsNotBlank())
+                    url += $"&after={afterId}";
+
+                // Get response
+                var content = await GetStringAsync(url);
+
+                // Parse
+                var users = JArray.Parse(content).Select(m => ParseUser(m["user"]));
+
+                // Add user to cache
+                foreach (var user in users)
+                    _userCache[user.Id] = user;
+
+                // Add users to list
+                string currentUserId = null;
+                foreach (var user in users)
+                {
+                    // Add user
+                    result.Add(user);
+                    if (currentUserId == null || BigInteger.Parse(user.Id) > BigInteger.Parse(currentUserId))
+                        currentUserId = user.Id;
+                }
+
+                // If no users - break
+                if (currentUserId == null)
+                    break;
+
+                // Otherwise offset the next request
+                afterId = currentUserId;
+            }
+
+            return result;
+        }
+
         public async Task<IReadOnlyList<Message>> GetChannelMessagesAsync(string token, string channelId,
             DateTime? from, DateTime? to)
         {
+            Channel channel = await GetChannelAsync(token, channelId);
+
+            try
+            {
+                await GetGuildRolesAsync(token, channel.GuildId);
+            }
+            catch (HttpErrorStatusCodeException e) { } // This will be thrown if the user doesnt have the MANAGE_ROLES permission for the guild
+
             var result = new List<Message>();
 
             // We are going backwards from last message to first
@@ -295,6 +529,13 @@ namespace DiscordChatExporter.Core.Services
             // Messages appear newest first, we need to reverse
             result.Reverse();
 
+            foreach (var message in result)
+            {
+                await FillMentionable(token, channel.GuildId, message);
+                foreach (var embed in message.Embeds)
+                    await FillMentionable(token, channel.GuildId, embed);
+            }
+
             return result;
         }
 

DiscordChatExporter.Core/Services/ExportService.Html.cs

@@ -3,17 +3,19 @@ using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using DiscordChatExporter.Core.Models;
 using Tyrrrz.Extensions;
+using System.Drawing;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace DiscordChatExporter.Core.Services
 {
     public partial class ExportService
     {
-        private string FormatMessageContentHtml(Message message)
+        private string MarkdownToHtml(string content, IMentionable mentionable = null, bool allowLinks = false)
         {
             // A lot of these regexes were inspired by or taken from MarkdownSharp
 
-            var content = message.Content;
-
             // HTML-encode content
             content = HtmlEncode(content);
 
@@ -29,7 +31,7 @@ namespace DiscordChatExporter.Core.Services
 
             // Encode URLs
             content = Regex.Replace(content,
-                @"((https?|ftp)://[-a-zA-Z0-9+&@#/%?=~_|!:,\.\[\]\(\);]*[-a-zA-Z0-9+&@#/%=~_|\[\])])(?=$|\W)",
+                @"(\b(?:(?:https?|ftp|file)://|www\.|ftp\.)(?:\([-a-zA-Z0-9+&@#/%?=~_|!:,\.\[\];]*\)|[-a-zA-Z0-9+&@#/%?=~_|!:,\.\[\];])*(?:\([-a-zA-Z0-9+&@#/%?=~_|!:,\.\[\];]*\)|[-a-zA-Z0-9+&@#/%=~_|$]))",
                 m => $"\x1AL{Base64Encode(m.Groups[1].Value)}\x1AL");
 
             // Process bold (**text**)
@@ -52,6 +54,12 @@ namespace DiscordChatExporter.Core.Services
             content = Regex.Replace(content, "\x1AI(.*?)\x1AI",
                 m => $"<span class=\"pre\">{Base64Decode(m.Groups[1].Value)}</span>");
 
+            if (allowLinks)
+            {
+                content = Regex.Replace(content, "\\[([^\\]]+)\\]\\(\x1AL(.*?)\x1AL\\)",
+                    m => $"<a href=\"{Base64Decode(m.Groups[2].Value)}\">{m.Groups[1].Value}</a>");
+            }
+
             // Decode and process URLs
             content = Regex.Replace(content, "\x1AL(.*?)\x1AL",
                 m => $"<a href=\"{Base64Decode(m.Groups[1].Value)}\">{Base64Decode(m.Groups[1].Value)}</a>");
@@ -65,31 +73,34 @@ namespace DiscordChatExporter.Core.Services
             // Meta mentions (@here)
             content = content.Replace("@here", "<span class=\"mention\">@here</span>");
 
-            // User mentions (<@id> and <@!id>)
-            foreach (var mentionedUser in message.MentionedUsers)
+            if (mentionable != null)
             {
-                content = Regex.Replace(content, $"&lt;@!?{mentionedUser.Id}&gt;",
-                    $"<span class=\"mention\" title=\"{HtmlEncode(mentionedUser.FullName)}\">" +
-                    $"@{HtmlEncode(mentionedUser.Name)}" +
-                    "</span>");
-            }
+                // User mentions (<@id> and <@!id>)
+                foreach (var mentionedUser in mentionable.MentionedUsers)
+                {
+                    content = Regex.Replace(content, $"&lt;@!?{mentionedUser.Id}&gt;",
+                        $"<span class=\"mention\" title=\"{HtmlEncode(mentionedUser.FullName)}\">" +
+                        $"@{HtmlEncode(mentionedUser.Name)}" +
+                        "</span>");
+                }
 
-            // Role mentions (<@&id>)
-            foreach (var mentionedRole in message.MentionedRoles)
-            {
-                content = content.Replace($"&lt;@&amp;{mentionedRole.Id}&gt;",
-                    "<span class=\"mention\">" +
-                    $"@{HtmlEncode(mentionedRole.Name)}" +
-                    "</span>");
-            }
+                // Role mentions (<@&id>)
+                foreach (var mentionedRole in mentionable.MentionedRoles)
+                {
+                    content = content.Replace($"&lt;@&amp;{mentionedRole.Id}&gt;",
+                        "<span class=\"mention\">" +
+                        $"@{HtmlEncode(mentionedRole.Name)}" +
+                        "</span>");
+                }
 
-            // Channel mentions (<#id>)
-            foreach (var mentionedChannel in message.MentionedChannels)
-            {
-                content = content.Replace($"&lt;#{mentionedChannel.Id}&gt;",
-                    "<span class=\"mention\">" +
-                    $"#{HtmlEncode(mentionedChannel.Name)}" +
-                    "</span>");
+                // Channel mentions (<#id>)
+                foreach (var mentionedChannel in mentionable.MentionedChannels)
+                {
+                    content = content.Replace($"&lt;#{mentionedChannel.Id}&gt;",
+                        "<span class=\"mention\">" +
+                        $"#{HtmlEncode(mentionedChannel.Name)}" +
+                        "</span>");
+                }
             }
 
             // Custom emojis (<:name:id>)
@@ -99,6 +110,145 @@ namespace DiscordChatExporter.Core.Services
             return content;
         }
 
+        private string FormatMessageContentHtml(Message message)
+        {
+            return MarkdownToHtml(message.Content, message);
+        }
+
+        // The code used to convert embeds to html was based heavily off of the Embed Visualizer project, from this file:
+        // https://github.com/leovoel/embed-visualizer/blob/master/src/components/embed.jsx
+
+        private string EmbedColorPillToHtml(Color? color)
+        {
+            string backgroundColor = "";
+
+            if (color != null)
+                backgroundColor = $"rgba({color?.R},{color?.G},{color?.B},1)";
+
+            return $"<div class='embed-color-pill' style='background-color: {backgroundColor}'></div>";
+        }
+
+        private string EmbedTitleToHtml(string title, string url)
+        {
+            if (title == null)
+                return null;
+
+            string computed = $"<div class='embed-title'>{MarkdownToHtml(title)}</div>";
+            if (url != null)
+                computed = $"<a target='_blank' rel='noreferrer' href='{url}' class='embed-title'>{MarkdownToHtml(title)}</a>";
+
+            return computed;
+        }
+
+        private string EmbedDescriptionToHtml(string content, IMentionable mentionable)
+        {
+            if (content == null)
+                return null;
+
+            return $"<div class='embed-description markup'>{MarkdownToHtml(content, mentionable, true)}</div>";
+        }
+
+        private string EmbedAuthorToHtml(string name, string url, string icon_url)
+        {
+            if (name == null)
+                return null;
+
+            string authorName = null;
+            if (name != null)
+            {
+                authorName = $"<span class='embed-author-name'>{name}</span>";
+                if (url != null)
+                    authorName = $"<a target='_blank' rel='noreferrer' href='{url}' class='embed-author-name'>{name}</a>";
+            }
+
+            string authorIcon = icon_url != null ? $"<img src='{icon_url}' role='presentation' class='embed-author-icon' />" : null;
+
+            return $"<div class='embed-author'>{authorIcon}{authorName}</div>";
+        }
+
+        private string EmbedFieldToHtml(string name, string value, bool? inline, IMentionable mentionable)
+        {
+            if (name == null && value == null)
+                return null;
+
+            string cls = "embed-field" + (inline == true ? " embed-field-inline" : "");
+
+            string fieldName = name != null ? $"<div class='embed-field-name'>{MarkdownToHtml(name)}</div>" : null;
+            string fieldValue = value != null ? $"<div class='embed-field-value markup'>{MarkdownToHtml(value, mentionable, true)}</div>" : null;
+
+            return $"<div class='{cls}'>{fieldName}{fieldValue}</div>";
+        }
+
+        private string EmbedThumbnailToHtml(string url)
+        {
+            if (url == null)
+                return null;
+
+            return $@"
+              <img
+                  src = '{url}'
+                  role = 'presentation'
+                  class='embed-rich-thumb'
+                  style='max-width: 80px; max-height: 80px'
+              />";
+        }
+
+        private string EmbedImageToHtml(string url)
+        {
+            if (url == null)
+                return null;
+
+            return $"<a class='embed-thumbnail embed-thumbnail-rich'><img class='image' role='presentation' src='{url}' /></a>";
+        }
+
+        private string EmbedFooterToHtml(DateTime? timestamp, string text, string icon_url)
+        {
+            if (text == null && timestamp == null)
+                return null;
+
+            // format: ddd MMM Do, YYYY [at] h:mm A
+
+            string time = timestamp != null ? HtmlEncode(timestamp?.ToString(_settingsService.DateFormat)) : null;
+
+            string footerText = string.Join(" | ", new List<string> { text, time }.Where(s => s != null));
+            string footerIcon = text != null && icon_url != null
+                ? $"<img src='{icon_url}' class='embed-footer-icon' role='presentation' width='20' height='20' />"
+                : null;
+
+            return $"<div>{footerIcon}<span class='embed-footer'>{footerText}</span></div>";
+        }
+
+        private string EmbedFieldsToHtml(IReadOnlyList<EmbedField> fields, IMentionable mentionable)
+        {
+            if (fields.Count == 0)
+                return null;
+
+            return $"<div class='embed-fields'>{string.Join("", fields.Select(f => EmbedFieldToHtml(f.Name, f.Value, f.Inline, mentionable)))}</div>";
+        }
+
+        private string FormatEmbedHtml(Embed embed)
+        {
+            return $@"
+                <div class='accessory'>
+                  <div class='embed-wrapper'>
+                    {EmbedColorPillToHtml(embed.Color)}
+                    <div class='embed embed-rich'>
+                      <div class='embed-content'>
+                        <div class='embed-content-inner'>
+                          {EmbedAuthorToHtml(embed.Author?.Name, embed.Author?.Url, embed.Author?.IconUrl)}
+                          {EmbedTitleToHtml(embed.Title, embed.Url)}
+                          {EmbedDescriptionToHtml(embed.Description, embed)}
+                          {EmbedFieldsToHtml(embed.Fields, embed)}
+                        </div>
+                        {EmbedThumbnailToHtml(embed.Thumbnail?.Url)}
+                      </div>
+                      {EmbedImageToHtml(embed.Image?.Url)}
+                      {EmbedFooterToHtml(embed.TimeStamp, embed.Footer?.Text, embed.Footer?.IconUrl)}
+                    </div>
+                  </div>
+                </div>";
+        }
+
         private async Task ExportAsHtmlAsync(ChannelChatLog log, TextWriter output, string css)
         {
             // Generation info
@@ -193,6 +343,13 @@ namespace DiscordChatExporter.Core.Services
                             await output.WriteLineAsync("</div>");
                         }
                     }
+
+                    // Embeds
+                    foreach (var embed in message.Embeds)
+                    {
+                        var contentFormatted = FormatEmbedHtml(embed);
+                        await output.WriteAsync(contentFormatted);
+                    }
                 }
 
                 await output.WriteLineAsync("</div>"); // msg-right

DiscordChatExporter.Core/Services/ExportService.cs

@@ -22,25 +22,25 @@ namespace DiscordChatExporter.Core.Services
         {
             using (var output = File.CreateText(filePath))
             {
+                var sharedCss = Assembly.GetExecutingAssembly()
+                        .GetManifestResourceString("DiscordChatExporter.Core.Resources.ExportService.Shared.css");
+
                 if (format == ExportFormat.PlainText)
                 {
                     await ExportAsPlainTextAsync(log, output);
                 }
-
                 else if (format == ExportFormat.HtmlDark)
                 {
                     var css = Assembly.GetExecutingAssembly()
                         .GetManifestResourceString("DiscordChatExporter.Core.Resources.ExportService.DarkTheme.css");
-                    await ExportAsHtmlAsync(log, output, css);
+                    await ExportAsHtmlAsync(log, output, $"{sharedCss}\n{css}");
                 }
-
                 else if (format == ExportFormat.HtmlLight)
                 {
                     var css = Assembly.GetExecutingAssembly()
                         .GetManifestResourceString("DiscordChatExporter.Core.Resources.ExportService.LightTheme.css");
-                    await ExportAsHtmlAsync(log, output, css);
+                    await ExportAsHtmlAsync(log, output, $"{sharedCss}\n{css}");
                 }
-
                 else if (format == ExportFormat.Csv)
                 {
                     await ExportAsCsvAsync(log, output);