diff --git a/LICENSE.django b/LICENSE.django new file mode 100644 index 0000000..7fd349d --- /dev/null +++ b/LICENSE.django @@ -0,0 +1,36 @@ +The Whatsapp Chat Exporter is licensed under the MIT license. For more information, +refer to the file LICENSE. + +Whatsapp Chat Exporter incorporates code from Django, governed by the three-clause +BSD license—a permissive open-source license. The copyright and license details are +provided below to adhere to Django's terms. + +------ + +Copyright (c) Django Software Foundation and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md index 83d1896..a5c6b10 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ usage: wtsexporter [-h] [-a] [-i] [-e EXPORTED] [-w WA] [-m MEDIA] [-b BACKUP] [ [-k KEY] [-t TEMPLATE] [-s] [-c] [--offline OFFLINE] [--size [SIZE]] [--no-html] [--check-update] [--assume-first-as-me] [--no-avatar] [--import] [--business] [--preserve-timestamp] [--wab WAB] [--time-offset {-12 to 14}] [--date DATE] [--date-format FORMAT] [--include [phone number ...]] - [--exclude [phone number ...]] + [--exclude [phone number ...]] [--create-separated-media] A customizable Android and iPhone WhatsApp database parser that will give you the history of your WhatsApp conversations in HTML and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported. @@ -164,8 +164,11 @@ options: Include chats that match the supplied phone number --exclude [phone number ...] Exclude chats that match the supplied phone number + --create-separated-media + Create a copy of the media seperated per chat in /separated/ directory + (Android only) -WhatsApp Chat Exporter: 0.9.7 Licensed with MIT +WhatsApp Chat Exporter: 0.10.0 Licensed with MIT ``` # To do diff --git a/Whatsapp_Chat_Exporter/__init__.py b/Whatsapp_Chat_Exporter/__init__.py index 9519d84..2765a6d 100644 --- a/Whatsapp_Chat_Exporter/__init__.py +++ b/Whatsapp_Chat_Exporter/__init__.py @@ -1,3 +1,3 @@ #!/usr/bin/python3 -__version__ = "0.9.7" +__version__ = "0.10.0" diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 89f6335..a332181 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -26,7 +26,8 @@ def main(): description = 'A customizable Android and iOS/iPadOS WhatsApp database parser that ' 'will give you the history of your WhatsApp conversations in HTML ' 'and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.', - epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT' + epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT. See' + 'https://wts.knugi.dev/docs?dest=osl for all open source licenses.' ) parser.add_argument( '-a', @@ -245,6 +246,13 @@ def main(): action='store_true', help="Output the JSON file per chat" ) + parser.add_argument( + "--create-separated-media", + dest="separate_media", + default=False, + action='store_true', + help="Create a copy of the media seperated per chat in /separated/ directory" + ) args = parser.parse_args() # Check for updates @@ -310,7 +318,6 @@ def main(): parser.error("Enter a phone number in the chat filter. See https://wts.knugi.dev/docs?dest=chat") filter_chat = (args.filter_chat_include, args.filter_chat_exclude) - data = {} if args.android: @@ -417,7 +424,7 @@ def main(): with sqlite3.connect(msg_db) as db: db.row_factory = sqlite3.Row messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat) - media(db, data, args.media, args.filter_date, filter_chat) + media(db, data, args.media, args.filter_date, filter_chat, args.separate_media) vcard(db, data, args.media, args.filter_date, filter_chat) if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 546bf4d..437fb8e 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -4,6 +4,7 @@ import sqlite3 import os import io import hmac +import shutil from pathlib import Path from mimetypes import MimeTypes from hashlib import sha256 @@ -12,7 +13,7 @@ from Whatsapp_Chat_Exporter.data_model import ChatStore, Message from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, determine_metadata, JidType from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location -from Whatsapp_Chat_Exporter.utility import get_chat_condition +from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify try: import zlib @@ -477,7 +478,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat): +def media(db, data, media_folder, filter_date, filter_chat, separate_media=True): # Get media c = db.cursor() try: @@ -569,6 +570,15 @@ def media(db, data, media_folder, filter_date, filter_chat): message.mime = "application/octet-stream" else: message.mime = content["mime_type"] + if separate_media: + chat_display_name = slugify(data[content["key_remote_jid"]].name or message.sender \ + or content["key_remote_jid"].split('@')[0], True) + current_filename = file_path.split("/")[-1] + new_folder = os.path.join(media_folder, "separated", chat_display_name) + Path(new_folder).mkdir(parents=True, exist_ok=True) + new_path = os.path.join(new_folder, current_filename) + shutil.copy2(file_path, new_path) + message.data = new_path else: if False: # Block execution try: diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index a1dc525..d2fc715 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -1,11 +1,12 @@ #!/usr/bin/python3 import os +import shutil from glob import glob from pathlib import Path from mimetypes import MimeTypes from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition, slugify def contacts(db, data): @@ -205,7 +206,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat): +def media(db, data, media_folder, filter_date, filter_chat, separate_media=False): c = db.cursor() # Get media c.execute(f"""SELECT count() @@ -256,6 +257,15 @@ def media(db, data, media_folder, filter_date, filter_chat): message.mime = "application/octet-stream" else: message.mime = content["ZVCARDSTRING"] + if separate_media: + chat_display_name = slugify(data[content["ZCONTACTJID"]].name or message.sender \ + or content["ZCONTACTJID"].split('@')[0], True) + current_filename = file_path.split("/")[-1] + new_folder = os.path.join(media_folder, "separated", chat_display_name) + Path(new_folder).mkdir(parents=True, exist_ok=True) + new_path = os.path.join(new_folder, current_filename) + shutil.copy2(file_path, new_path) + message.data = new_path else: if False: # Block execution try: diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index ba2787b..fc75ae8 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -3,6 +3,8 @@ import json import os from bleach import clean as sanitize from markupsafe import Markup +import unicodedata +import re from datetime import datetime from enum import IntEnum from Whatsapp_Chat_Exporter.data_model import ChatStore @@ -309,6 +311,23 @@ def setup_template(template, no_avatar): APPLE_TIME = datetime.timestamp(datetime(2001, 1, 1)) +def slugify(value, allow_unicode=False): + """ + Taken from https://github.com/django/django/blob/master/django/utils/text.py + Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated + dashes to single dashes. Remove characters that aren't alphanumerics, + underscores, or hyphens. Convert to lowercase. Also strip leading and + trailing whitespace, dashes, and underscores. + """ + value = str(value) + if allow_unicode: + value = unicodedata.normalize('NFKC', value) + else: + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value.lower()) + return re.sub(r'[-\s]+', '-', value).strip('-_') + + class WhatsAppIdentifier(StrEnum): MESSAGE = "7c7fba66680ef796b916b067077cc246adacf01d" CONTACT = "b8548dc30aa1030df0ce18ef08b882cf7ab5212f" diff --git a/docs.html b/docs.html index 634f9a8..7b2e841 100644 --- a/docs.html +++ b/docs.html @@ -7,6 +7,7 @@ "filter": "Filter", "date": "Filters#date-filters", "chat": "Filters#chat-filter", + "osl": "Open-Source-Licenses" null: "" }; const dest = new URLSearchParams(window.location.search).get('dest');