From 520e023aff48be49a822ef7f24d0fe5065d1a6db Mon Sep 17 00:00:00 2001 From: Andrew Kolos Date: Fri, 23 Oct 2020 09:38:15 -0400 Subject: [PATCH] Stop redundantly downloading media when re-exporting (#395) --- .../Commands/Base/ExportCommandBase.cs | 4 ++ .../Base/ExportMultipleCommandBase.cs | 1 + .../Exporting/ExportContext.cs | 2 +- .../Exporting/ExportRequest.cs | 4 ++ .../Exporting/MediaDownloader.cs | 41 ++++++++++++++++--- .../Services/SettingsService.cs | 2 + .../ViewModels/Dialogs/SettingsViewModel.cs | 6 +++ .../ViewModels/RootViewModel.cs | 1 + .../Views/Dialogs/SettingsView.xaml | 17 +++++++- 9 files changed, 70 insertions(+), 8 deletions(-) diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index c4309ddd..4aa49b88 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -29,6 +29,9 @@ namespace DiscordChatExporter.Cli.Commands.Base [CommandOption("media", Description = "Download referenced media content.")] public bool ShouldDownloadMedia { get; set; } + [CommandOption("reuse-media", Description = "If the media folder already exists, reuse media inside it to skip downloads.")] + public bool ShouldReuseMedia { get; set; } + [CommandOption("dateformat", Description = "Date format used in output.")] public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt"; @@ -48,6 +51,7 @@ namespace DiscordChatExporter.Cli.Commands.Base Before, PartitionLimit, ShouldDownloadMedia, + ShouldReuseMedia, DateFormat ); diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs index 01670814..716ca22b 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs @@ -48,6 +48,7 @@ namespace DiscordChatExporter.Cli.Commands.Base Before, PartitionLimit, ShouldDownloadMedia, + ShouldReuseMedia, DateFormat ); diff --git a/DiscordChatExporter.Domain/Exporting/ExportContext.cs b/DiscordChatExporter.Domain/Exporting/ExportContext.cs index 68a44329..c8e58a47 100644 --- a/DiscordChatExporter.Domain/Exporting/ExportContext.cs +++ b/DiscordChatExporter.Domain/Exporting/ExportContext.cs @@ -34,7 +34,7 @@ namespace DiscordChatExporter.Domain.Exporting Channels = channels; Roles = roles; - _mediaDownloader = new MediaDownloader(request.OutputMediaDirPath); + _mediaDownloader = new MediaDownloader(request.OutputMediaDirPath, request.ShouldReuseMedia); } public string FormatDate(DateTimeOffset date) => Request.DateFormat switch diff --git a/DiscordChatExporter.Domain/Exporting/ExportRequest.cs b/DiscordChatExporter.Domain/Exporting/ExportRequest.cs index cddcd87c..745ba361 100644 --- a/DiscordChatExporter.Domain/Exporting/ExportRequest.cs +++ b/DiscordChatExporter.Domain/Exporting/ExportRequest.cs @@ -30,6 +30,8 @@ namespace DiscordChatExporter.Domain.Exporting public bool ShouldDownloadMedia { get; } + public bool ShouldReuseMedia { get; } + public string DateFormat { get; } public ExportRequest( @@ -41,6 +43,7 @@ namespace DiscordChatExporter.Domain.Exporting DateTimeOffset? before, int? partitionLimit, bool shouldDownloadMedia, + bool shouldReuseMedia, string dateFormat) { Guild = guild; @@ -51,6 +54,7 @@ namespace DiscordChatExporter.Domain.Exporting Before = before; PartitionLimit = partitionLimit; ShouldDownloadMedia = shouldDownloadMedia; + ShouldReuseMedia = shouldReuseMedia; DateFormat = dateFormat; OutputBaseFilePath = GetOutputBaseFilePath( diff --git a/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs b/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs index e8f1e7ea..3dff9588 100644 --- a/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs +++ b/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs @@ -3,6 +3,8 @@ using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using DiscordChatExporter.Domain.Internal; @@ -16,13 +18,16 @@ namespace DiscordChatExporter.Domain.Exporting { private readonly HttpClient _httpClient = Singleton.HttpClient; private readonly string _workingDirPath; + + private readonly bool _reuseMedia; private readonly AsyncRetryPolicy _httpRequestPolicy; private readonly Dictionary _pathMap = new Dictionary(); - public MediaDownloader(string workingDirPath) + public MediaDownloader(string workingDirPath, bool reuseMedia) { _workingDirPath = workingDirPath; + _reuseMedia = reuseMedia; _httpRequestPolicy = Policy .Handle() @@ -37,11 +42,18 @@ namespace DiscordChatExporter.Domain.Exporting return cachedFilePath; var fileName = GetFileNameFromUrl(url); - var filePath = PathEx.MakeUniqueFilePath(Path.Combine(_workingDirPath, fileName)); + var filePath = Path.Combine(_workingDirPath, fileName); - Directory.CreateDirectory(_workingDirPath); + if (!_reuseMedia) + { + filePath = PathEx.MakeUniqueFilePath(filePath); + } - await _httpClient.DownloadAsync(url, filePath); + if (!_reuseMedia || !File.Exists(filePath)) + { + Directory.CreateDirectory(_workingDirPath); + await _httpClient.DownloadAsync(url, filePath); + } return _pathMap[url] = filePath; }); @@ -50,6 +62,23 @@ namespace DiscordChatExporter.Domain.Exporting internal partial class MediaDownloader { + private static int URL_HASH_LENGTH = 5; + private static string HashUrl(string url) + { + using (var md5 = MD5.Create()) + { + var inputBytes = Encoding.UTF8.GetBytes(url); + var hashBytes = md5.ComputeHash(inputBytes); + + var hashBuilder = new StringBuilder(); + for (int i = 0; i < hashBytes.Length; i++) + { + hashBuilder.Append(hashBytes[i].ToString("X2")); + } + return hashBuilder.ToString().Truncate(URL_HASH_LENGTH); + } + } + private static string GetRandomFileName() => Guid.NewGuid().ToString().Replace("-", "").Substring(0, 16); private static string GetFileNameFromUrl(string url) @@ -57,10 +86,10 @@ namespace DiscordChatExporter.Domain.Exporting var originalFileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value; var fileName = !string.IsNullOrWhiteSpace(originalFileName) - ? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(50)}{Path.GetExtension(originalFileName)}" + ? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(42)}-({HashUrl(url)}){Path.GetExtension(originalFileName)}" : GetRandomFileName(); return PathEx.EscapePath(fileName); } } -} \ No newline at end of file +} diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index 6a3ecfd8..14afc674 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -16,6 +16,8 @@ namespace DiscordChatExporter.Gui.Services public int ParallelLimit { get; set; } = 1; + public bool ShouldReuseMedia { get; set; } = false; + public AuthToken? LastToken { get; set; } public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark; diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs index 6a5a76f1..bf1ebf2b 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs @@ -37,6 +37,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs get => _settingsService.ParallelLimit; set => _settingsService.ParallelLimit = value.Clamp(1, 10); } + + public bool ShouldReuseMedia + { + get => _settingsService.ShouldReuseMedia; + set => _settingsService.ShouldReuseMedia = value; + } public SettingsViewModel(SettingsService settingsService) { diff --git a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs index 4f442d03..13721c88 100644 --- a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs @@ -204,6 +204,7 @@ namespace DiscordChatExporter.Gui.ViewModels dialog.Before, dialog.PartitionLimit, dialog.ShouldDownloadMedia, + _settingsService.ShouldReuseMedia, _settingsService.DateFormat ); diff --git a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml index d01de4d9..d67cb22c 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml @@ -7,7 +7,7 @@ xmlns:materialDesign="http://materialdesigninxaml.net/winfx/xaml/themes" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:s="https://github.com/canton7/Stylet" - Width="300" + Width="310" d:DataContext="{d:DesignInstance Type=dialogs:SettingsViewModel}" Style="{DynamicResource MaterialDesignRoot}" mc:Ignorable="d"> @@ -65,6 +65,21 @@ IsChecked="{Binding IsTokenPersisted}" /> + + + + + +