calvin.erfmann/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-02-01 15:49:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
Tyrrrz
2021-06-16 16:52:18 +03:00
parent 9491e18e2f
commit 29552be6e5
21 changed files with 116 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DiscordChatExporter.Core/Discord/AuthToken.cs
View File

@@ -2,8 +2,6 @@
namespace DiscordChatExporter.Core.Discord
{
public enum AuthTokenType { User, Bot }
public class AuthToken
{
public AuthTokenType Type { get; }

8
DiscordChatExporter.Core/Discord/AuthTokenType.cs Normal file
View File

@@ -0,0 +1,8 @@
namespace DiscordChatExporter.Core.Discord
{
public enum AuthTokenType
{
User,
Bot
}
}

42
DiscordChatExporter.Core/Discord/Data/Channel.cs
View File

@@ -7,32 +7,19 @@ using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#channel-object-channel-types
// Order of enum fields needs to match the order in the docs.
public enum ChannelType
{
GuildTextChat,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
}
// https://discord.com/developers/docs/resources/channel#channel-object
public partial class Channel : IHasId, IHasPosition
public partial class Channel : IHasId
{
public Snowflake Id { get; }
public ChannelType Type { get; }
public bool IsTextChannel =>
Type == ChannelType.GuildTextChat ||
Type == ChannelType.DirectTextChat ||
Type == ChannelType.DirectGroupTextChat ||
Type == ChannelType.GuildNews ||
Type == ChannelType.GuildStore;
public bool IsTextChannel => Type is
ChannelType.GuildTextChat or
ChannelType.DirectTextChat or
ChannelType.DirectGroupTextChat or
ChannelType.GuildNews or
ChannelType.GuildStore;
public Snowflake GuildId { get; }
@@ -48,7 +35,7 @@ namespace DiscordChatExporter.Core.Discord.Data
Snowflake id,
ChannelType type,
Snowflake guildId,
ChannelCategory? category,
ChannelCategory category,
string name,
int? position,
string? topic)
@@ -56,14 +43,13 @@ namespace DiscordChatExporter.Core.Discord.Data
Id = id;
Type = type;
GuildId = guildId;
Category = category ?? GetFallbackCategory(type);
Category = category;
Name = name;
Position = position;
Topic = topic;
}
public override string ToString() => Name;
}
public partial class Channel
@@ -79,7 +65,7 @@ namespace DiscordChatExporter.Core.Discord.Data
ChannelType.GuildStore => "Store",
_ => "Default"
},
0
null
);
public static Channel Parse(JsonElement json, ChannelCategory? category = null, int? position = null)
@@ -87,23 +73,23 @@ namespace DiscordChatExporter.Core.Discord.Data
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")?.GetString().Pipe(Snowflake.Parse);
var topic = json.GetPropertyOrNull("topic")?.GetString();
var type = (ChannelType) json.GetProperty("type").GetInt32();
var name =
// Guild channel
json.GetPropertyOrNull("name")?.GetString() ??
// DM channel
json.GetPropertyOrNull("recipients")?.EnumerateArray().Select(User.Parse).Select(u => u.Name).JoinToString(", ") ??
// Fallback
id.ToString();
position ??= json.GetPropertyOrNull("position")?.GetInt32();
return new Channel(
id,
type,
guildId ?? Guild.DirectMessages.Id,
category ?? GetFallbackCategory(type),
name,
position,
position ?? json.GetPropertyOrNull("position")?.GetInt32(),
topic
);
}

16
DiscordChatExporter.Core/Discord/Data/ChannelCategory.cs
View File

@@ -1,13 +1,11 @@
using System.Linq;
using System.Text.Json;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Discord.Data
{
public partial class ChannelCategory : IHasId, IHasPosition
public partial class ChannelCategory : IHasId
{
public Snowflake Id { get; }
@@ -23,7 +21,6 @@ namespace DiscordChatExporter.Core.Discord.Data
}
public override string ToString() => Name;
}
public partial class ChannelCategory
@@ -31,19 +28,18 @@ namespace DiscordChatExporter.Core.Discord.Data
public static ChannelCategory Parse(JsonElement json, int? position = null)
{
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
position ??= json.GetPropertyOrNull("position")?.GetInt32();
var name = json.GetPropertyOrNull("name")?.GetString() ??
json.GetPropertyOrNull("recipients")?.EnumerateArray().Select(User.Parse).Select(u => u.Name).JoinToString(", ") ??
var name =
json.GetPropertyOrNull("name")?.GetString() ??
id.ToString();
return new ChannelCategory(
id,
name,
position
position ?? json.GetPropertyOrNull("position")?.GetInt32()
);
}
public static ChannelCategory Empty { get; } = new(Snowflake.Zero, "<unknown category>", 0);
}
}
}

15
DiscordChatExporter.Core/Discord/Data/ChannelType.cs Normal file
View File

@@ -0,0 +1,15 @@
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#channel-object-channel-types
// Order of enum fields needs to match the order in the docs.
public enum ChannelType
{
GuildTextChat = 0,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
}
}

7
DiscordChatExporter.Core/Discord/Data/Common/IHasPosition.cs
View File

@@ -1,7 +0,0 @@
namespace DiscordChatExporter.Core.Discord.Data.Common
{
public interface IHasPosition
{
int? Position { get; }
}
}

2
DiscordChatExporter.Core/Discord/Data/Embed.cs
View File

@@ -63,10 +63,10 @@ namespace DiscordChatExporter.Core.Discord.Data
public static Embed Parse(JsonElement json)
{
var title = json.GetPropertyOrNull("title")?.GetString();
var description = json.GetPropertyOrNull("description")?.GetString();
var url = json.GetPropertyOrNull("url")?.GetString();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffset();
var color = json.GetPropertyOrNull("color")?.GetInt32().Pipe(System.Drawing.Color.FromArgb).ResetAlpha();
var description = json.GetPropertyOrNull("description")?.GetString();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);

18
DiscordChatExporter.Core/Discord/Data/Emoji.cs
View File

@@ -18,13 +18,12 @@ namespace DiscordChatExporter.Core.Discord.Data
public string ImageUrl { get; }
public Emoji(string? id, string name, bool isAnimated)
public Emoji(string? id, string name, bool isAnimated, string imageUrl)
{
Id = id;
Name = name;
IsAnimated = isAnimated;
ImageUrl = GetImageUrl(id, name, isAnimated);
ImageUrl = imageUrl;
}
public override string ToString() => Name;
@@ -53,12 +52,9 @@ namespace DiscordChatExporter.Core.Discord.Data
// Custom emoji
if (!string.IsNullOrWhiteSpace(id))
{
// Animated
if (isAnimated)
return $"https://cdn.discordapp.com/emojis/{id}.gif";
// Non-animated
return $"https://cdn.discordapp.com/emojis/{id}.png";
return isAnimated
? $"https://cdn.discordapp.com/emojis/{id}.gif"
: $"https://cdn.discordapp.com/emojis/{id}.png";
}
// Standard emoji
@@ -73,7 +69,9 @@ namespace DiscordChatExporter.Core.Discord.Data
var name = json.GetProperty("name").GetString();
var isAnimated = json.GetPropertyOrNull("animated")?.GetBoolean() ?? false;
return new Emoji(id, name, isAnimated);
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(id, name, isAnimated, imageUrl);
}
}
}

10
DiscordChatExporter.Core/Discord/Data/Member.cs
View File

@@ -19,10 +19,10 @@ namespace DiscordChatExporter.Core.Discord.Data
public IReadOnlyList<Snowflake> RoleIds { get; }
public Member(User user, string? nick, IReadOnlyList<Snowflake> roleIds)
public Member(User user, string nick, IReadOnlyList<Snowflake> roleIds)
{
User = user;
Nick = nick ?? user.Name;
Nick = nick;
RoleIds = roleIds;
}
@@ -33,7 +33,7 @@ namespace DiscordChatExporter.Core.Discord.Data
{
public static Member CreateForUser(User user) => new(
user,
null,
user.Name,
Array.Empty<Snowflake>()
);
@@ -43,12 +43,12 @@ namespace DiscordChatExporter.Core.Discord.Data
var nick = json.GetPropertyOrNull("nick")?.GetString();
var roleIds =
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString().Pipe(Snowflake.Parse)).ToArray() ??
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).Select(Snowflake.Parse).ToArray() ??
Array.Empty<Snowflake>();
return new Member(
user,
nick,
nick ?? user.Name,
roleIds
);
}

2
DiscordChatExporter.Core/Discord/Data/Message.cs
View File

@@ -100,7 +100,7 @@ namespace DiscordChatExporter.Core.Discord.Data
var type = (MessageType) json.GetProperty("type").GetInt32();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBoolean() ?? false;
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Message.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var content = type switch
{

2
DiscordChatExporter.Core/Discord/Data/Reaction.cs
View File

@@ -23,8 +23,8 @@ namespace DiscordChatExporter.Core.Discord.Data
{
public static Reaction Parse(JsonElement json)
{
var count = json.GetProperty("count").GetInt32();
var emoji = json.GetProperty("emoji").Pipe(Emoji.Parse);
var count = json.GetProperty("count").GetInt32();
return new Reaction(emoji, count);
}

3
DiscordChatExporter.Core/Discord/Data/Role.cs
View File

@@ -40,7 +40,8 @@ namespace DiscordChatExporter.Core.Discord.Data
var name = json.GetProperty("name").GetString();
var position = json.GetProperty("position").GetInt32();
var color = json.GetPropertyOrNull("color")?
var color = json
.GetPropertyOrNull("color")?
.GetInt32()
.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha()

2
DiscordChatExporter.Core/Discord/Data/User.cs
View File

@@ -58,10 +58,10 @@ namespace DiscordChatExporter.Core.Discord.Data
public static User Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
var discriminator = json.GetProperty("discriminator").GetString().Pipe(int.Parse);
var name = json.GetProperty("username").GetString();
var avatarHash = json.GetProperty("avatar").GetString();
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
var avatarUrl = !string.IsNullOrWhiteSpace(avatarHash)
? GetAvatarUrl(id, avatarHash)

35
DiscordChatExporter.Core/Discord/DiscordClient.cs
View File

@@ -11,7 +11,6 @@ using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Http;
using JsonExtensions.Reading;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Discord
{
@@ -29,7 +28,9 @@ namespace DiscordChatExporter.Core.Discord
}
public DiscordClient(AuthToken token)
: this(Http.Client, token) {}
: this(Http.Client, token)
{
}
private async ValueTask<HttpResponseMessage> GetResponseAsync(string url) =>
await Http.ResponsePolicy.ExecuteAsync(async () =>
@@ -64,7 +65,7 @@ namespace DiscordChatExporter.Core.Discord
return response.IsSuccessStatusCode
? await response.Content.ReadAsJsonAsync()
: (JsonElement?) null;
: null;
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync()
@@ -118,29 +119,30 @@ namespace DiscordChatExporter.Core.Discord
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels");
var orderedResponse = response
var responseOrdered = response
.EnumerateArray()
.OrderBy(j => j.GetProperty("position").GetInt32())
.ThenBy(j => ulong.Parse(j.GetProperty("id").GetString()))
.ThenBy(j => Snowflake.Parse(j.GetProperty("id").GetString()))
.ToArray();
var categories = orderedResponse
.Where(j => j.GetProperty("type").GetInt32() == (int)ChannelType.GuildCategory)
var categories = responseOrdered
.Where(j => j.GetProperty("type").GetInt32() == (int) ChannelType.GuildCategory)
.Select((j, index) => ChannelCategory.Parse(j, index + 1))
.ToDictionary(j => j.Id.ToString());
.ToDictionary(j => j.Id.ToString(), StringComparer.Ordinal);
var position = 0;
foreach (var channelJson in orderedResponse)
foreach (var channelJson in responseOrdered)
{
var parentId = channelJson.GetPropertyOrNull("parent_id")?.GetString();
var category = !string.IsNullOrWhiteSpace(parentId)
? categories.GetValueOrDefault(parentId)
: null;
var channel = Channel.Parse(channelJson, category, position);
// Skip non-text channels
// We are only interested in text channels
if (!channel.IsTextChannel)
continue;
@@ -178,15 +180,12 @@ namespace DiscordChatExporter.Core.Discord
var response = await GetJsonResponseAsync($"channels/{channelId}");
return ChannelCategory.Parse(response);
}
/***
* In some cases, the Discord API returns an empty body when requesting some channel category info.
* Instead, we use an empty channel category as a fallback.
*/
// In some cases, the Discord API returns an empty body when requesting channel category.
// Instead, we use an empty channel category as a fallback.
catch (DiscordChatExporterException)
{
return ChannelCategory.Empty;
}
}
public async ValueTask<Channel> GetChannelAsync(Snowflake channelId)
@@ -221,7 +220,7 @@ namespace DiscordChatExporter.Core.Discord
IProgress<double>? progress = null)
{
// Get the last message in the specified range.
// This snapshots the boundaries, which means that messages posted after the exported started
// This snapshots the boundaries, which means that messages posted after the export started
// will not appear in the output.
// Additionally, it provides the date of the last message, which is used to calculate progress.
var lastMessage = await TryGetLastMessageAsync(channelId, before);
@@ -271,7 +270,7 @@ namespace DiscordChatExporter.Core.Discord
progress.Report(exportedDuration / totalDuration);
}
// Avoid division by zero if all messages have the exact same timestamp
// (which can happen easily if there's only one message in the channel)
// (which may be the case if there's only one message in the channel)
else
{
progress.Report(1);
@@ -284,4 +283,4 @@ namespace DiscordChatExporter.Core.Discord
}
}
}
}
}

11
DiscordChatExporter.Core/Discord/Snowflake.cs
View File

@@ -10,8 +10,9 @@ namespace DiscordChatExporter.Core.Discord
public Snowflake(ulong value) => Value = value;
public DateTimeOffset ToDate() =>
DateTimeOffset.FromUnixTimeMilliseconds((long) ((Value >> 22) + 1420070400000UL)).ToLocalTime();
public DateTimeOffset ToDate() => DateTimeOffset.FromUnixTimeMilliseconds(
(long) ((Value >> 22) + 1420070400000UL)
).ToLocalTime();
public override string ToString() => Value.ToString(CultureInfo.InvariantCulture);
}
@@ -53,9 +54,11 @@ namespace DiscordChatExporter.Core.Discord
public static Snowflake Parse(string str) => Parse(str, null);
}
public partial struct Snowflake : IEquatable<Snowflake>
public partial struct Snowflake : IComparable<Snowflake>, IEquatable<Snowflake>
{
public bool Equals(Snowflake other) => Value == other.Value;
public int CompareTo(Snowflake other) => Value.CompareTo(other.Value);
public bool Equals(Snowflake other) => CompareTo(other) == 0;
public override bool Equals(object? obj) => obj is Snowflake other && Equals(other);

4
DiscordChatExporter.Core/Exporting/ExportContext.cs
View File

@@ -79,7 +79,7 @@ namespace DiscordChatExporter.Core.Exporting
: filePath;
// HACK: for HTML, we need to format the URL properly
if (Request.Format == ExportFormat.HtmlDark || Request.Format == ExportFormat.HtmlLight)
if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight)
{
// Need to escape each path segment while keeping the directory separators intact
return relativeFilePath
@@ -93,7 +93,7 @@ namespace DiscordChatExporter.Core.Exporting
// Try to catch only exceptions related to failed HTTP requests
// https://github.com/Tyrrrz/DiscordChatExporter/issues/332
// https://github.com/Tyrrrz/DiscordChatExporter/issues/372
catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException)
catch (Exception ex) when (ex is HttpRequestException or OperationCanceledException)
{
// TODO: add logging so we can be more liberal with catching exceptions
// We don't want this to crash the exporting process in case of failure

4
DiscordChatExporter.Core/Exporting/Writers/Html/MessageGroup.cs
View File

@@ -36,9 +36,13 @@ namespace DiscordChatExporter.Core.Exporting.Writers.Html
internal partial class MessageGroup
{
public static bool CanJoin(Message message1, Message message2) =>
// Must be from the same author
message1.Author.Id == message2.Author.Id &&
// Author's name must not have changed between messages
string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal) &&
// Duration between messages must be 7 minutes or less
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7 &&
// Other message must not be a reply
message2.Reference is null;
public static MessageGroup Join(IReadOnlyList<Message> messages)

4
DiscordChatExporter.Core/Exporting/Writers/JsonMessageWriter.cs
View File

@@ -20,6 +20,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
});
}
@@ -300,4 +302,4 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await base.DisposeAsync();
}
}
}
}

2
DiscordChatExporter.Core/Utils/Extensions/GenericExtensions.cs
View File

@@ -9,6 +9,6 @@ namespace DiscordChatExporter.Core.Utils.Extensions
public static T? NullIf<T>(this T value, Func<T, bool> predicate) where T : struct =>
!predicate(value)
? value
: (T?) null;
: null;
}
}

7
DiscordChatExporter.Core/Utils/Extensions/StringExtensions.cs
View File

@@ -4,9 +4,14 @@ namespace DiscordChatExporter.Core.Utils.Extensions
{
public static class StringExtensions
{
public static string? NullIfWhiteSpace(this string str) =>
!string.IsNullOrWhiteSpace(str)
? str
: null;
public static string Truncate(this string str, int charCount) =>
str.Length > charCount
? str.Substring(0, charCount)
? str[..charCount]
: str;
public static StringBuilder AppendIfNotEmpty(this StringBuilder builder, char value) =>

24
DiscordChatExporter.Core/Utils/Http.cs
View File

@@ -5,6 +5,7 @@ using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils.Extensions;
using Polly;
namespace DiscordChatExporter.Core.Utils
@@ -41,21 +42,24 @@ namespace DiscordChatExporter.Core.Utils
},
(_, _, _, _) => Task.CompletedTask);
private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex)
{
private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
// This is extremely frail, but there's no other way
var statusCodeRaw = Regex.Match(ex.Message, @": (\d+) \(").Groups[1].Value;
return !string.IsNullOrWhiteSpace(statusCodeRaw)
? (HttpStatusCode) int.Parse(statusCodeRaw, CultureInfo.InvariantCulture)
: (HttpStatusCode?) null;
}
Regex
.Match(ex.Message, @": (\d+) \(")
.Groups[1]
.Value
.NullIfWhiteSpace()?
.Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
public static IAsyncPolicy ExceptionPolicy { get; } =
Policy
.Handle<IOException>() // dangerous
.Or<HttpRequestException>(ex => TryGetStatusCodeFromException(ex) == HttpStatusCode.TooManyRequests)
.Or<HttpRequestException>(ex => TryGetStatusCodeFromException(ex) == HttpStatusCode.RequestTimeout)
.Or<HttpRequestException>(ex => TryGetStatusCodeFromException(ex) >= HttpStatusCode.InternalServerError)
.Or<HttpRequestException>(ex =>
TryGetStatusCodeFromException(ex) is
HttpStatusCode.TooManyRequests or
HttpStatusCode.RequestTimeout or
HttpStatusCode.InternalServerError
)
.WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
}
}