From afa6052a08c1300a482005ffcc6a14faf9ab0ad7 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:39:19 +0800 Subject: [PATCH 01/23] Add note --- Whatsapp_Chat_Exporter/utility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 9f8c45c..3a02411 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -13,6 +13,7 @@ try: from enum import StrEnum, IntEnum except ImportError: # < Python 3.11 + # This should be removed when the support for Python 3.10 ends. from enum import Enum class StrEnum(str, Enum): pass From fa37dd4b2d90361e203e1cad5660802dcf5e6ac7 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:51:00 +0800 Subject: [PATCH 02/23] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5fa5484..f02e8c1 100644 --- a/setup.py +++ b/setup.py @@ -32,10 +32,10 @@ setuptools.setup( }, classifiers=[ "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Development Status :: 4 - Beta", @@ -45,7 +45,7 @@ setuptools.setup( "Topic :: Utilities", "Topic :: Database" ], - python_requires='>=3.8', + python_requires='>=3.9', install_requires=[ 'jinja2', 'bleach' From fef9684189d47b5cbbbc98d666a77f2e785c7b78 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:35:54 +0800 Subject: [PATCH 03/23] Remove __version__ Use importlib.metadata.version instead --- Whatsapp_Chat_Exporter/__init__.py | 3 --- Whatsapp_Chat_Exporter/__main__.py | 7 ++----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__init__.py b/Whatsapp_Chat_Exporter/__init__.py index 5060413..e69de29 100644 --- a/Whatsapp_Chat_Exporter/__init__.py +++ b/Whatsapp_Chat_Exporter/__init__.py @@ -1,3 +0,0 @@ -#!/usr/bin/python3 - -__version__ = "0.10.5" diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index bf5d963..7f7b7ca 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -22,10 +22,7 @@ from Whatsapp_Chat_Exporter.utility import check_update, import_from_json, sanit from argparse import ArgumentParser, SUPPRESS from datetime import datetime from sys import exit -try: - from .__init__ import __version__ -except ImportError: - from Whatsapp_Chat_Exporter.__init__ import __version__ +import importlib.metadata def main(): @@ -33,7 +30,7 @@ def main(): description = 'A customizable Android and iOS/iPadOS WhatsApp database parser that ' 'will give you the history of your WhatsApp conversations in HTML ' 'and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.', - epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT. See ' + epilog = f'WhatsApp Chat Exporter: {importlib.metadata.version("whatsapp_chat_exporter")} Licensed with MIT. See ' 'https://wts.knugi.dev/docs?dest=osl for all open source licenses.' ) parser.add_argument( From 209d5a7796baf8f066e9b110f16955cdaa51bec9 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:36:22 +0800 Subject: [PATCH 04/23] Migrate to pyproject.toml --- pyproject.toml | 61 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 70 -------------------------------------------------- 2 files changed, 61 insertions(+), 70 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e442fe0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "whatsapp-chat-exporter" +version = "0.10.5" +description = "A Whatsapp database parser that provides history of your Whatsapp conversations in HTML and JSON. Android, iOS, iPadOS, Crypt12, Crypt14, Crypt15 supported." +readme = "README.md" +authors = [ + { name = "KnugiHK", email = "hello@knugi.com" } +] +license = { text = "MIT" } +keywords = [ + "android", "ios", "parsing", "history", "iphone", "message", "crypt15", + "customizable", "whatsapp", "android-backup", "messages", "crypt14", + "crypt12", "whatsapp-chat-exporter", "whatsapp-export", "iphone-backup", + "whatsapp-database", "whatsapp-database-parser", "whatsapp-conversations" +] +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "Topic :: Communications :: Chat", + "Topic :: Utilities", + "Topic :: Database" +] +requires-python = ">=3.9" +dependencies = [ + "jinja2", + "bleach" +] + +[project.optional-dependencies] +android_backup = ["pycryptodome", "javaobj-py3"] +crypt12 = ["pycryptodome"] +crypt14 = ["pycryptodome"] +crypt15 = ["pycryptodome", "javaobj-py3"] +all = ["pycryptodome", "javaobj-py3", "vobject"] +everything = ["pycryptodome", "javaobj-py3", "vobject"] +backup = ["pycryptodome", "javaobj-py3"] +vcards = ["vobject", "pycryptodome", "javaobj-py3"] + +[project.scripts] +wtsexporter = "Whatsapp_Chat_Exporter.__main__:main" +waexporter = "Whatsapp_Chat_Exporter.__main__:main" +whatsapp-chat-exporter = "Whatsapp_Chat_Exporter.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["Whatsapp_Chat_Exporter"] + +[tool.setuptools.package-data] +template = ["whatsapp.html"] diff --git a/setup.py b/setup.py deleted file mode 100644 index f02e8c1..0000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -import setuptools -from re import search - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open("Whatsapp_Chat_Exporter/__init__.py", encoding="utf8") as f: - version = search(r'__version__ = "(.*?)"', f.read()).group(1) - -setuptools.setup( - name="whatsapp-chat-exporter", - version=version, - author="KnugiHK", - author_email="hello@knugi.com", - description=("A Whatsapp database parser that will give you the " - "history of your Whatsapp conversations in HTML and JSON. " - "Android, iOS, iPadOS, Crypt12, Crypt14, Crypt15 supported."), - long_description=long_description, - long_description_content_type="text/markdown", - license="MIT", - keywords=[ - "android", "ios", "parsing", "history", "iphone", "message", "crypt15", - "customizable", "whatsapp", "android-backup", "messages", "crypt14", - "crypt12", "whatsapp-chat-exporter", "whatsapp-export", "iphone-backup", - "whatsapp-database", "whatsapp-database-parser", "whatsapp-conversations" - ], - platforms=["any"], - url="https://github.com/KnugiHK/Whatsapp-Chat-Exporter", - packages=setuptools.find_packages(), - package_data={ - '': ['whatsapp.html'] - }, - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Topic :: Communications :: Chat", - "Topic :: Utilities", - "Topic :: Database" - ], - python_requires='>=3.9', - install_requires=[ - 'jinja2', - 'bleach' - ], - extras_require={ - 'android_backup': ["pycryptodome", "javaobj-py3"], - 'crypt12': ["pycryptodome"], - 'crypt14': ["pycryptodome"], - 'crypt15': ["pycryptodome", "javaobj-py3"], - 'all': ["pycryptodome", "javaobj-py3", "vobject"], - 'everything': ["pycryptodome", "javaobj-py3", "vobject"], - 'backup': ["pycryptodome", "javaobj-py3"], - 'vcards': ["vobject", "pycryptodome", "javaobj-py3"], - }, - entry_points={ - "console_scripts": [ - "wtsexporter = Whatsapp_Chat_Exporter.__main__:main", - "waexporter = Whatsapp_Chat_Exporter.__main__:main", - "whatsapp-chat-exporter = Whatsapp_Chat_Exporter.__main__:main" - ] - } -) From 82d24857786c7fd518a8c9a7d4375b4b66e68213 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:42:33 +0800 Subject: [PATCH 05/23] Fixed the incorrect iOS timestamp #124 --- Whatsapp_Chat_Exporter/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 3a02411..589c751 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -391,7 +391,7 @@ def setup_template(template, no_avatar): return template_env.get_template(template_file) # iOS Specific -APPLE_TIME = datetime.timestamp(datetime(2001, 1, 1)) +APPLE_TIME = 978307200 def slugify(value, allow_unicode=False): From a8bac8837ed43ba128a752a583265f588156698a Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:57:38 +0800 Subject: [PATCH 06/23] Automatically detect timezone offset when --time-offset is not provided #124 --- Whatsapp_Chat_Exporter/android_handler.py | 6 +++--- Whatsapp_Chat_Exporter/ios_handler.py | 4 ++-- Whatsapp_Chat_Exporter/utility.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index b10c40c..30cc2d9 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -12,7 +12,7 @@ from hashlib import sha256 from base64 import b64decode, b64encode from datetime import datetime from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata +from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template, JidType from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, chat_is_empty @@ -354,7 +354,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): timestamp=content["timestamp"], time=content["timestamp"], key_id=content["key_id"], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET ) if isinstance(content["data"], bytes): message.data = ("The message is binary data and its base64 is " @@ -717,7 +717,7 @@ def calls(db, data, timezone_offset, filter_chat): timestamp=content["timestamp"], time=content["timestamp"], key_id=content["call_id"], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET ) _jid = content["raw_string"] name = data[_jid].name if _jid in data else content["chat_subject"] or None diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 0744f4f..566dd41 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -7,7 +7,7 @@ from pathlib import Path from mimetypes import MimeTypes from markupsafe import escape as htmle from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition, slugify +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, CURRENT_TZ_OFFSET, Device, get_chat_condition, slugify def contacts(db, data): @@ -149,7 +149,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): timestamp=ts, time=ts, # TODO: Could be bug key_id=content["ZSTANZAID"][:17], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET ) invalid = False if is_group_message and content["ZISFROMME"] == 0: diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 589c751..e94303e 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -23,6 +23,7 @@ except ImportError: MAX_SIZE = 4 * 1024 * 1024 # Default 4MB ROW_SIZE = 0x3D0 +CURRENT_TZ_OFFSET = datetime.now().astimezone().utcoffset().seconds / 3600 def convert_time_unit(time_second: int): From a1f6320cd8159fa528bea5e2f68cb2350e581056 Mon Sep 17 00:00:00 2001 From: Knugi <24708955+KnugiHK@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:12:26 +0000 Subject: [PATCH 07/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a2ae5b..fd31d75 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ cd working_wts ``` > [!TIP] -> macOS users should grant Full Disk Access to Terminal in the Security & Privacy settings before using the exporter. +> macOS users should grant *Full Disk Access* to Terminal in the *Security & Privacy* settings before using the exporter. ## Working with Android ### Unencrypted WhatsApp database From 7117716e5b6480851f7090e393715714f1a73d4f Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 19 Dec 2024 19:13:21 +0800 Subject: [PATCH 08/23] Add crypt14 offset --- Whatsapp_Chat_Exporter/utility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index e94303e..76816b2 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -259,6 +259,7 @@ CRYPT14_OFFSETS = ( {"iv": 66, "db": 99}, {"iv": 67, "db": 193}, {"iv": 67, "db": 194}, + {"iv": 67, "db": 158}, ) From a0b81671213d6b6d989ad79987a195d8a33d85f9 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:01:25 +0800 Subject: [PATCH 09/23] Create a whatsapp-alike theme #97 --- Whatsapp_Chat_Exporter/__main__.py | 21 +- Whatsapp_Chat_Exporter/android_handler.py | 5 +- Whatsapp_Chat_Exporter/utility.py | 6 +- Whatsapp_Chat_Exporter/whatsapp_new.html | 368 ++++++++++++++++++++++ 4 files changed, 391 insertions(+), 9 deletions(-) create mode 100644 Whatsapp_Chat_Exporter/whatsapp_new.html diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 7f7b7ca..4a2600b 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -296,7 +296,15 @@ def main(): default=None, type=str, const="result", - help="Export chats in text format similar to what WhatsApp officially provided (default if present: result/)") + help="Export chats in text format similar to what WhatsApp officially provided (default if present: result/)" + ) + parser.add_argument( + "--experimental-new-theme", + dest="whatsapp_theme", + default=False, + action='store_true', + help="Use the newly designed WhatsApp-alike theme" + ) args = parser.parse_args() @@ -358,6 +366,8 @@ def main(): args.filter_date = f"<= {_timestamp - APPLE_TIME}" else: parser.error("Unsupported date format. See https://wts.knugi.dev/docs?dest=date") + if args.whatsapp_theme: + args.template = "whatsapp_new.html" if args.filter_chat_include is not None and args.filter_chat_exclude is not None: parser.error("Chat inclusion and exclusion filters cannot be used together.") if args.filter_chat_include is not None: @@ -502,7 +512,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.filter_empty, + args.whatsapp_theme ) else: print( @@ -539,7 +550,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.filter_empty, + args.whatsapp_theme ) for file in glob.glob(r'*.*'): shutil.copy(file, args.output) @@ -553,7 +565,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.filter_empty, + args.whatsapp_theme ) if args.text_format: diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 30cc2d9..44a3724 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -760,9 +760,10 @@ def create_html( offline_static=False, maximum_size=None, no_avatar=False, - filter_empty=True + filter_empty=True, + experimental=False ): - template = setup_template(template, no_avatar) + template = setup_template(template, no_avatar, experimental) total_row_number = len(data) print(f"\nGenerating chats...(0/{total_row_number})", end="\r") diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 76816b2..46826b3 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -376,10 +376,10 @@ def get_status_location(output_folder, offline_static): w3css = os.path.join(offline_static, "w3.css") -def setup_template(template, no_avatar): - if template is None: +def setup_template(template, no_avatar, experimental=False): + if template is None or experimental: template_dir = os.path.dirname(__file__) - template_file = "whatsapp.html" + template_file = "whatsapp.html" if not experimental else template else: template_dir = os.path.dirname(template) template_file = os.path.basename(template) diff --git a/Whatsapp_Chat_Exporter/whatsapp_new.html b/Whatsapp_Chat_Exporter/whatsapp_new.html new file mode 100644 index 0000000..787a81d --- /dev/null +++ b/Whatsapp_Chat_Exporter/whatsapp_new.html @@ -0,0 +1,368 @@ + + + + Whatsapp - {{ name }} + + + + + + + + +
+
+
+
+ {% if not no_avatar %} +
+ {% if their_avatar is not none %} + + {% else %} + + {% endif %} +
+ {% endif %} +
+

Chat history with {{ name }}

+ {% if status is not none %}

{{ status }}

{% endif %} +
+
+
+ + + {% if next %} + + + + + + {% endif %} +
+ +
+ + +
+
+
+
+
+ + {% set last = {'last': 946688461.001} %} + {% for msg in msgs -%} + {% if determine_day(last.last, msg.timestamp) is not none %} +
+
+ {{ determine_day(last.last, msg.timestamp) }} +
+
+ {% if last.update({'last': msg.timestamp}) %}{% endif %} + {% endif %} + + {% if msg.from_me == true %} +
+
+ {% if msg.reply is not none %} + +
+

Replying to

+

+ {% if msg.quoted_data is not none %} + "{{msg.quoted_data}}" + {% else %} + this message + {% endif %} +

+
+
+ {% endif %} +

+ {% if msg.meta == true or msg.media == false and msg.data is none %} +

+
+ {% if msg.safe %} + {{ msg.data | safe or 'Not supported WhatsApp internal message' }} + {% else %} + {{ msg.data or 'Not supported WhatsApp internal message' }} + {% endif %} +
+
+ {% if msg.caption is not none %} +

{{ msg.caption | urlize(none, true, '_blank') }}

+ {% endif %} + {% else %} + {% if msg.media == false %} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} + {% else %} + {% if "image/" in msg.mime %} + + + + {% elif "audio/" in msg.mime %} + + {% elif "video/" in msg.mime %} + + {% elif "/" in msg.mime %} + The file cannot be displayed here, however it should be located at here + {% else %} + {% filter escape %}{{ msg.data }}{% endfilter %} + {% endif %} + {% if msg.caption is not none %} + {{ msg.caption | urlize(none, true, '_blank') }} + {% endif %} + {% endif %} + {% endif %} +

+

{{ msg.time }}

+
+
+ {% else %} +
+
+ {% if msg.reply is not none %} + +
+

Replying to

+

+ {% if msg.quoted_data is not none %} + {{msg.quoted_data}} + {% else %} + this message + {% endif %} +

+
+
+ {% endif %} +

+ {% if msg.meta == true or msg.media == false and msg.data is none %} +

+
+ {% if msg.safe %} + {{ msg.data | safe or 'Not supported WhatsApp internal message' }} + {% else %} + {{ msg.data or 'Not supported WhatsApp internal message' }} + {% endif %} +
+
+ {% if msg.caption is not none %} +

{{ msg.caption | urlize(none, true, '_blank') }}

+ {% endif %} + {% else %} + {% if msg.media == false %} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} + {% else %} + {% if "image/" in msg.mime %} + + + + {% elif "audio/" in msg.mime %} + + {% elif "video/" in msg.mime %} + + {% elif "/" in msg.mime %} + The file cannot be displayed here, however it should be located at here + {% else %} + {% filter escape %}{{ msg.data }}{% endfilter %} + {% endif %} + {% if msg.caption is not none %} + {{ msg.caption | urlize(none, true, '_blank') }} + {% endif %} + {% endif %} + {% endif %} +

+
+ + {% if msg.sender is not none %} + {{ msg.sender }} + {% endif %} + + + {{ msg.time }} +
+
+
+ {% endif %} + {% endfor %} +
+
+
+ + + \ No newline at end of file From cf03bfba1b7bfc3e61862a4f5bb0f55314028a74 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:30:11 +0800 Subject: [PATCH 10/23] Bug fix on duplicated base name #126 --- Whatsapp_Chat_Exporter/ios_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 566dd41..96fdced 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -292,7 +292,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False Path(new_folder).mkdir(parents=True, exist_ok=True) new_path = os.path.join(new_folder, current_filename) shutil.copy2(file_path, new_path) - message.data = new_path + message.data = '/'.join(new_path.split("\\")[1:]) else: message.data = "The media is missing" message.mime = "media" From 7a1fa463685cc696a12fb945d88b4bd0d6112734 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 2 Jan 2025 20:48:11 +0800 Subject: [PATCH 11/23] Implement call log for iOS #122 --- Whatsapp_Chat_Exporter/__main__.py | 13 ++++ Whatsapp_Chat_Exporter/ios_handler.py | 72 ++++++++++++++++++++- Whatsapp_Chat_Exporter/ios_media_handler.py | 10 +++ Whatsapp_Chat_Exporter/utility.py | 1 + 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 4a2600b..fcf231c 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -305,6 +305,15 @@ def main(): action='store_true', help="Use the newly designed WhatsApp-alike theme" ) + parser.add_argument( + "--call-db", + dest="call_db_ios", + nargs='?', + default=None, + type=str, + const="1b432994e958845fffe8e2f190f26d1511534088", + help="Path to call database (default: 1b432994e958845fffe8e2f190f26d1511534088) iOS only" + ) args = parser.parse_args() @@ -500,6 +509,10 @@ def main(): vcard(db, data, args.media, args.filter_date, filter_chat) if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) + elif args.ios and args.call_db_ios is not None: + with sqlite3.connect(args.call_db_ios) as cdb: + cdb.row_factory = sqlite3.Row + ios_handler.calls(cdb, data, args.timezone_offset, filter_chat) if not args.no_html: if args.enrich_from_vcards is not None and not contact_store.is_empty(): contact_store.enrich_from_vcards(data) diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 96fdced..a0296a7 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -7,7 +7,8 @@ from pathlib import Path from mimetypes import MimeTypes from markupsafe import escape as htmle from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, CURRENT_TZ_OFFSET, Device, get_chat_condition, slugify +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, CURRENT_TZ_OFFSET, get_chat_condition +from Whatsapp_Chat_Exporter.utility import bytes_to_readable, convert_time_unit, slugify, Device def contacts(db, data): @@ -361,3 +362,72 @@ def vcard(db, data, media_folder, filter_date, filter_chat): message.meta = True message.safe = True print(f"Processing vCards...({index + 1}/{total_row_number})", end="\r") + + +def calls(db, data, timezone_offset, filter_chat): + c = db.cursor() + c.execute(f"""SELECT count() + FROM ZWACDCALLEVENT + WHERE 1=1 + {get_chat_condition(filter_chat[0], True, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")} + {get_chat_condition(filter_chat[1], False, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")}""") + total_row_number = c.fetchone()[0] + if total_row_number == 0: + return + print(f"\nProcessing calls...({total_row_number})", end="\r") + c.execute(f"""SELECT ZCALLIDSTRING, + ZGROUPCALLCREATORUSERJIDSTRING, + ZGROUPJIDSTRING, + ZDATE, + ZOUTCOME, + ZBYTESRECEIVED + ZBYTESSENT AS bytes_transferred, + ZDURATION, + ZVIDEO, + ZMISSED, + ZINCOMING + FROM ZWACDCALLEVENT + INNER JOIN ZWAAGGREGATECALLEVENT + ON ZWACDCALLEVENT.Z1CALLEVENTS = ZWAAGGREGATECALLEVENT.Z_PK + WHERE 1=1 + {get_chat_condition(filter_chat[0], True, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")} + {get_chat_condition(filter_chat[1], False, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")}""") + chat = ChatStore(Device.ANDROID, "WhatsApp Calls") + content = c.fetchone() + while content is not None: + ts = APPLE_TIME + int(content["ZDATE"]) + call = Message( + from_me=content["ZINCOMING"] == 0, + timestamp=ts, + time=ts, + key_id=content["ZCALLIDSTRING"], + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET + ) + _jid = content["ZGROUPCALLCREATORUSERJIDSTRING"] + name = data[_jid].name if _jid in data else None + if _jid is not None and "@" in _jid: + fallback = _jid.split('@')[0] + else: + fallback = None + call.sender = name or fallback + call.meta = True + call.data = ( + f"A {'video' if content['ZVIDEO'] == 1 else 'voice'} " + f"call {'to' if call.from_me else 'from'} " + f"{call.sender} was " + ) + if content['ZOUTCOME'] in (1, 4): + call.data += "not answered." if call.from_me else "missed." + elif content['ZOUTCOME'] == 2: + call.data += "failed." + elif content['ZOUTCOME'] == 0: + call_time = convert_time_unit(int(content['ZDURATION'])) + call_bytes = bytes_to_readable(content['bytes_transferred']) + call.data += ( + f"initiated and lasted for {call_time} " + f"with {call_bytes} data transferred." + ) + else: + call.data += "in an unknown state." + chat.add_message(call.key_id, call) + content = c.fetchone() + data["000000000000000"] = chat \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/ios_media_handler.py b/Whatsapp_Chat_Exporter/ios_media_handler.py index c4360af..dc817b6 100644 --- a/Whatsapp_Chat_Exporter/ios_media_handler.py +++ b/Whatsapp_Chat_Exporter/ios_media_handler.py @@ -35,6 +35,11 @@ def extract_encrypted(base_dir, password, identifiers, decrypt_chunk_size): domain_like=identifiers.DOMAIN, output_filename=identifiers.CONTACT ) + backup.extract_file( + relative_path=RelativePath.WHATSAPP_CALLS, + domain_like=identifiers.DOMAIN, + output_filename=identifiers.CALL + ) except ValueError: print("Failed to decrypt backup: incorrect password?") exit(7) @@ -87,6 +92,7 @@ def extract_media(base_dir, identifiers, decrypt_chunk_size): else: wts_db = os.path.join(base_dir, identifiers.MESSAGE[:2], identifiers.MESSAGE) contact_db = os.path.join(base_dir, identifiers.CONTACT[:2], identifiers.CONTACT) + call_db = os.path.join(base_dir, identifiers.CALL[:2], identifiers.CALL) if not os.path.isfile(wts_db): if identifiers is WhatsAppIdentifier: print("WhatsApp database not found.") @@ -99,6 +105,10 @@ def extract_media(base_dir, identifiers, decrypt_chunk_size): print("Contact database not found. Skipping...") else: shutil.copyfile(contact_db, identifiers.CONTACT) + if not os.path.isfile(call_db): + print("Call database not found. Skipping...") + else: + shutil.copyfile(call_db, identifiers.CALL) _wts_id = identifiers.DOMAIN with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: manifest.row_factory = sqlite3.Row diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 46826b3..0a72c74 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -416,6 +416,7 @@ def slugify(value, allow_unicode=False): class WhatsAppIdentifier(StrEnum): MESSAGE = "7c7fba66680ef796b916b067077cc246adacf01d" CONTACT = "b8548dc30aa1030df0ce18ef08b882cf7ab5212f" + CALL = "1b432994e958845fffe8e2f190f26d1511534088" DOMAIN = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" From 92d710bce81dcecec05c68f7ba7cd254e2e26809 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Thu, 2 Jan 2025 20:57:28 +0800 Subject: [PATCH 12/23] Differentiate group and personal calls --- Whatsapp_Chat_Exporter/ios_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index a0296a7..8034eba 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -411,7 +411,8 @@ def calls(db, data, timezone_offset, filter_chat): call.sender = name or fallback call.meta = True call.data = ( - f"A {'video' if content['ZVIDEO'] == 1 else 'voice'} " + f"A {'group ' if content['ZGROUPJIDSTRING'] is not None else ''}" + f"{'video' if content['ZVIDEO'] == 1 else 'voice'} " f"call {'to' if call.from_me else 'from'} " f"{call.sender} was " ) From 23af55d6454886114ad5883b5812399ba03f96d7 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 4 Jan 2025 18:18:34 +0800 Subject: [PATCH 13/23] Implement empty chat filtering from SQL #112 This commit also removed the old empty chat filtering logic. --- Whatsapp_Chat_Exporter/__main__.py | 20 ++++++-------- Whatsapp_Chat_Exporter/android_handler.py | 33 ++++++++++++++++------- Whatsapp_Chat_Exporter/ios_handler.py | 6 ++--- Whatsapp_Chat_Exporter/utility.py | 10 +++---- 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index fcf231c..31ef6cf 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -17,8 +17,8 @@ else: from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler from Whatsapp_Chat_Exporter.data_model import ChatStore -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, chat_is_empty, readable_to_bytes -from Whatsapp_Chat_Exporter.utility import check_update, import_from_json, sanitize_filename, bytes_to_readable +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, readable_to_bytes, check_update +from Whatsapp_Chat_Exporter.utility import import_from_json, sanitize_filename, bytes_to_readable from argparse import ArgumentParser, SUPPRESS from datetime import datetime from sys import exit @@ -254,7 +254,9 @@ def main(): dest="filter_empty", default=True, action='store_false', - help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." + help=("By default, the exporter will not render chats with no valid message. " + "Setting this flag will cause the exporter to render those. " + "This is useful if chat(s) are missing from the output") ) parser.add_argument( "--per-chat", @@ -504,9 +506,9 @@ def main(): if os.path.isfile(msg_db): with sqlite3.connect(msg_db) as db: db.row_factory = sqlite3.Row - messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat) - media(db, data, args.media, args.filter_date, filter_chat, args.separate_media) - vcard(db, data, args.media, args.filter_date, filter_chat) + messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat, args.filter_empty) + media(db, data, args.media, args.filter_date, filter_chat, args.filter_empty, args.separate_media) + vcard(db, data, args.media, args.filter_date, filter_chat, args.filter_empty) if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) elif args.ios and args.call_db_ios is not None: @@ -525,7 +527,6 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty, args.whatsapp_theme ) else: @@ -563,7 +564,6 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty, args.whatsapp_theme ) for file in glob.glob(r'*.*'): @@ -578,7 +578,6 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty, args.whatsapp_theme ) @@ -587,9 +586,6 @@ def main(): android_handler.create_txt(data, args.text_format) if args.json and not args.import_json: - if args.filter_empty: - data = {k: v for k, v in data.items() if not chat_is_empty(v)} - if args.enrich_from_vcards is not None and not contact_store.is_empty(): contact_store.enrich_from_vcards(data) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 44a3724..38a3e3c 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -12,10 +12,10 @@ from hashlib import sha256 from base64 import b64decode, b64encode from datetime import datetime from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata -from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template, JidType +from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata, get_cond_for_empty +from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location -from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, chat_is_empty +from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, JidType try: import zlib @@ -173,7 +173,7 @@ def contacts(db, data): row = c.fetchone() -def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): +def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty): # Get message history c = db.cursor() try: @@ -181,7 +181,10 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): FROM messages INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")} {f'AND timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")}""") @@ -196,6 +199,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")} {f'AND timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") @@ -253,6 +257,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN receipt_user ON receipt_user.message_row_id = messages._id WHERE messages.key_remote_jid <> '-1' + {get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} @@ -321,6 +326,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN receipt_user ON receipt_user.message_row_id = message._id WHERE key_remote_jid <> '-1' + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} @@ -488,7 +494,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat, separate_media=True): +def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=True): # Get media c = db.cursor() try: @@ -498,7 +504,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON message_media.message_row_id = messages._id INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}""") @@ -514,6 +523,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") @@ -536,7 +546,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON message_media.file_hash = media_hash_thumbnail.media_hash INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE jid.type <> 7 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} @@ -563,6 +576,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE jid.type <> 7 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} @@ -613,7 +627,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data, media_folder, filter_date, filter_chat): +def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty): c = db.cursor() try: c.execute(f"""SELECT message_row_id, @@ -625,7 +639,10 @@ def vcard(db, data, media_folder, filter_date, filter_chat): ON messages_vcards.message_row_id = messages._id INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} @@ -646,6 +663,7 @@ def vcard(db, data, media_folder, filter_date, filter_chat): LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} @@ -760,7 +778,6 @@ def create_html( offline_static=False, maximum_size=None, no_avatar=False, - filter_empty=True, experimental=False ): template = setup_template(template, no_avatar, experimental) @@ -775,8 +792,6 @@ def create_html( for current, contact in enumerate(data): chat = data[contact] - if filter_empty and chat_is_empty(chat): - continue safe_file_name, name = get_file_name(contact, chat) if maximum_size is not None: diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 8034eba..61964c7 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -27,7 +27,7 @@ def contacts(db, data): content = c.fetchone() -def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): +def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty): c = db.cursor() # Get contacts c.execute( @@ -227,7 +227,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat, separate_media=False): +def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=False): c = db.cursor() # Get media c.execute(f"""SELECT count() @@ -308,7 +308,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data, media_folder, filter_date, filter_chat): +def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty): c = db.cursor() c.execute(f"""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, ZWAMEDIAITEM.ZMESSAGE, diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 0a72c74..d7de1a4 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -220,6 +220,10 @@ def get_file_name(contact: str, chat: ChatStore): return sanitize_filename(file_name), name +def get_cond_for_empty(enable, jid_field: str, broadcast_field: str): + return f"AND (chat.sort_timestamp IS NOT NULL OR {jid_field}='status@broadcast' OR {broadcast_field}>0)" if enable else "" + + def get_chat_condition(filter, include, columns, jid=None, platform=None): if filter is not None: conditions = [] @@ -245,12 +249,6 @@ def get_chat_condition(filter, include, columns, jid=None, platform=None): else: return "" -def _is_message_empty(message): - return (message.data is None or message.data == "") and not message.media - -def chat_is_empty(chat: ChatStore): - return len(chat.messages) == 0 or all(_is_message_empty(message) for message in chat.messages.values()) - # Android Specific CRYPT14_OFFSETS = ( From bf993c53029f15aaf6ecad419c3d92b177cb682d Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 9 Feb 2025 12:47:35 +0800 Subject: [PATCH 14/23] Change the column to determine if the chat should be filtered (#112) --- Whatsapp_Chat_Exporter/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index d7de1a4..0a3c7f9 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -221,7 +221,7 @@ def get_file_name(contact: str, chat: ChatStore): def get_cond_for_empty(enable, jid_field: str, broadcast_field: str): - return f"AND (chat.sort_timestamp IS NOT NULL OR {jid_field}='status@broadcast' OR {broadcast_field}>0)" if enable else "" + return f"AND (chat.hidden=0 OR {jid_field}='status@broadcast' OR {broadcast_field}>0)" if enable else "" def get_chat_condition(filter, include, columns, jid=None, platform=None): From f300e017edb2fc0f2081c1aed26c6ce1e3be8488 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 9 Feb 2025 13:23:23 +0800 Subject: [PATCH 15/23] Implement lazy loading for video (#103) --- Whatsapp_Chat_Exporter/whatsapp.html | 39 ++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/Whatsapp_Chat_Exporter/whatsapp.html b/Whatsapp_Chat_Exporter/whatsapp.html index 20caf30..63700e8 100644 --- a/Whatsapp_Chat_Exporter/whatsapp.html +++ b/Whatsapp_Chat_Exporter/whatsapp.html @@ -20,7 +20,6 @@ } footer { border-top: 2px solid #e3e6e7; - font-size: 2em; padding: 20px 0 20px 0; } article { @@ -156,8 +155,8 @@ {% elif "video/" in msg.mime %} -