diff --git a/README.md b/README.md index 013f796..00325c3 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ cd working_wts ``` > [!TIP] -> macOS users should grant Full Disk Access to Terminal in the Security & Privacy settings before using the exporter. +> macOS users should grant *Full Disk Access* to Terminal in the *Security & Privacy* settings before using the exporter. ## Working with Android ### Unencrypted WhatsApp database diff --git a/Whatsapp_Chat_Exporter/__init__.py b/Whatsapp_Chat_Exporter/__init__.py index 5060413..e69de29 100644 --- a/Whatsapp_Chat_Exporter/__init__.py +++ b/Whatsapp_Chat_Exporter/__init__.py @@ -1,3 +0,0 @@ -#!/usr/bin/python3 - -__version__ = "0.10.5" diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index bf5d963..ac1d3dc 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -17,15 +17,12 @@ else: from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler from Whatsapp_Chat_Exporter.data_model import ChatStore -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, chat_is_empty, readable_to_bytes -from Whatsapp_Chat_Exporter.utility import check_update, import_from_json, sanitize_filename, bytes_to_readable +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, readable_to_bytes, check_update +from Whatsapp_Chat_Exporter.utility import import_from_json, sanitize_filename, bytes_to_readable from argparse import ArgumentParser, SUPPRESS from datetime import datetime from sys import exit -try: - from .__init__ import __version__ -except ImportError: - from Whatsapp_Chat_Exporter.__init__ import __version__ +import importlib.metadata def main(): @@ -33,7 +30,7 @@ def main(): description = 'A customizable Android and iOS/iPadOS WhatsApp database parser that ' 'will give you the history of your WhatsApp conversations in HTML ' 'and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.', - epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT. See ' + epilog = f'WhatsApp Chat Exporter: {importlib.metadata.version("whatsapp_chat_exporter")} Licensed with MIT. See ' 'https://wts.knugi.dev/docs?dest=osl for all open source licenses.' ) parser.add_argument( @@ -257,7 +254,9 @@ def main(): dest="filter_empty", default=True, action='store_false', - help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." + help=("By default, the exporter will not render chats with no valid message. " + "Setting this flag will cause the exporter to render those. " + "This is useful if chat(s) are missing from the output") ) parser.add_argument( "--per-chat", @@ -299,7 +298,30 @@ def main(): default=None, type=str, const="result", - help="Export chats in text format similar to what WhatsApp officially provided (default if present: result/)") + help="Export chats in text format similar to what WhatsApp officially provided (default if present: result/)" + ) + parser.add_argument( + "--experimental-new-theme", + dest="whatsapp_theme", + default=False, + action='store_true', + help="Use the newly designed WhatsApp-alike theme" + ) + parser.add_argument( + "--call-db", + dest="call_db_ios", + nargs='?', + default=None, + type=str, + const="1b432994e958845fffe8e2f190f26d1511534088", + help="Path to call database (default: 1b432994e958845fffe8e2f190f26d1511534088) iOS only" + ) + parser.add_argument( + "--headline", + dest="headline", + default="Chat history with ??", + help="The custom headline for the HTML output. Use '??' as a placeholder for the chat name" + ) args = parser.parse_args() @@ -320,6 +342,8 @@ def main(): parser.error("JSON file not found.") if args.android and args.business: parser.error("WhatsApp Business is only available on iOS for now.") + if "??" not in args.headline: + parser.error("--headline must contain '??' for replacement.") if args.json_per_chat and ( (args.json[-5:] != ".json" and os.path.isfile(args.json)) or \ (args.json[-5:] == ".json" and os.path.isfile(args.json[:-5])) @@ -361,6 +385,8 @@ def main(): args.filter_date = f"<= {_timestamp - APPLE_TIME}" else: parser.error("Unsupported date format. See https://wts.knugi.dev/docs?dest=date") + if args.whatsapp_theme: + args.template = "whatsapp_new.html" if args.filter_chat_include is not None and args.filter_chat_exclude is not None: parser.error("Chat inclusion and exclusion filters cannot be used together.") if args.filter_chat_include is not None: @@ -488,11 +514,15 @@ def main(): if os.path.isfile(msg_db): with sqlite3.connect(msg_db) as db: db.row_factory = sqlite3.Row - messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat) - media(db, data, args.media, args.filter_date, filter_chat, args.separate_media) - vcard(db, data, args.media, args.filter_date, filter_chat) + messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat, args.filter_empty) + media(db, data, args.media, args.filter_date, filter_chat, args.filter_empty, args.separate_media) + vcard(db, data, args.media, args.filter_date, filter_chat, args.filter_empty) if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) + elif args.ios and args.call_db_ios is not None: + with sqlite3.connect(args.call_db_ios) as cdb: + cdb.row_factory = sqlite3.Row + ios_handler.calls(cdb, data, args.timezone_offset, filter_chat) if not args.no_html: if args.enrich_from_vcards is not None and not contact_store.is_empty(): contact_store.enrich_from_vcards(data) @@ -505,7 +535,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.whatsapp_theme, + args.headline ) else: print( @@ -542,7 +573,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.whatsapp_theme, + args.headline ) for file in glob.glob(r'*.*'): shutil.copy(file, args.output) @@ -556,7 +588,8 @@ def main(): args.offline, args.size, args.no_avatar, - args.filter_empty + args.whatsapp_theme, + args.headline ) if args.text_format: @@ -564,9 +597,6 @@ def main(): android_handler.create_txt(data, args.text_format) if args.json and not args.import_json: - if args.filter_empty: - data = {k: v for k, v in data.items() if not chat_is_empty(v)} - if args.enrich_from_vcards is not None and not contact_store.is_empty(): contact_store.enrich_from_vcards(data) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index b10c40c..0713232 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -12,10 +12,10 @@ from hashlib import sha256 from base64 import b64decode, b64encode from datetime import datetime from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata -from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template, JidType +from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata, get_cond_for_empty +from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location -from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, chat_is_empty +from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, JidType try: import zlib @@ -173,7 +173,7 @@ def contacts(db, data): row = c.fetchone() -def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): +def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty): # Get message history c = db.cursor() try: @@ -181,7 +181,10 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): FROM messages INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")} {f'AND timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")}""") @@ -196,6 +199,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")} {f'AND timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") @@ -253,6 +257,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN receipt_user ON receipt_user.message_row_id = messages._id WHERE messages.key_remote_jid <> '-1' + {get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} @@ -321,6 +326,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN receipt_user ON receipt_user.message_row_id = message._id WHERE key_remote_jid <> '-1' + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} @@ -354,7 +360,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): timestamp=content["timestamp"], time=content["timestamp"], key_id=content["key_id"], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET, + message_type=content["media_wa_type"] ) if isinstance(content["data"], bytes): message.data = ("The message is binary data and its base64 is " @@ -488,7 +495,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat, separate_media=True): +def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=True): # Get media c = db.cursor() try: @@ -498,7 +505,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON message_media.message_row_id = messages._id INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}""") @@ -514,6 +524,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") @@ -536,7 +547,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON message_media.file_hash = media_hash_thumbnail.media_hash INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE jid.type <> 7 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} @@ -563,6 +577,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE jid.type <> 7 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} @@ -613,7 +628,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data, media_folder, filter_date, filter_chat): +def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty): c = db.cursor() try: c.execute(f"""SELECT message_row_id, @@ -625,7 +640,10 @@ def vcard(db, data, media_folder, filter_date, filter_chat): ON messages_vcards.message_row_id = messages._id INNER JOIN jid ON messages.key_remote_jid = jid.raw_string + LEFT JOIN chat + ON chat.jid_row_id = jid._id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} @@ -646,6 +664,7 @@ def vcard(db, data, media_folder, filter_date, filter_chat): LEFT JOIN jid jid_group ON jid_group._id = message.sender_jid_row_id WHERE 1=1 + {get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} @@ -717,7 +736,7 @@ def calls(db, data, timezone_offset, filter_chat): timestamp=content["timestamp"], time=content["timestamp"], key_id=content["call_id"], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET ) _jid = content["raw_string"] name = data[_jid].name if _jid in data else content["chat_subject"] or None @@ -760,9 +779,10 @@ def create_html( offline_static=False, maximum_size=None, no_avatar=False, - filter_empty=True + experimental=False, + headline=None ): - template = setup_template(template, no_avatar) + template = setup_template(template, no_avatar, experimental) total_row_number = len(data) print(f"\nGenerating chats...(0/{total_row_number})", end="\r") @@ -774,8 +794,6 @@ def create_html( for current, contact in enumerate(data): chat = data[contact] - if filter_empty and chat_is_empty(chat): - continue safe_file_name, name = get_file_name(contact, chat) if maximum_size is not None: @@ -799,8 +817,10 @@ def create_html( render_box, contact, w3css, - f"{safe_file_name}-{current_page + 1}.html", - chat + chat, + headline, + next=f"{safe_file_name}-{current_page + 1}.html", + previous=f"{safe_file_name}-{current_page - 1}.html" if current_page > 1 else False ) render_box = [message] current_size = 0 @@ -819,8 +839,10 @@ def create_html( render_box, contact, w3css, + chat, + headline, False, - chat + previous=f"{safe_file_name}-{current_page - 1}.html" ) else: output_file_name = f"{output_folder}/{safe_file_name}.html" @@ -831,8 +853,9 @@ def create_html( chat.get_messages(), contact, w3css, - False, - chat + chat, + headline, + False ) if current % 10 == 0: print(f"Generating chats...({current}/{total_row_number})", end="\r") diff --git a/Whatsapp_Chat_Exporter/data_model.py b/Whatsapp_Chat_Exporter/data_model.py index 68dad15..f08acca 100644 --- a/Whatsapp_Chat_Exporter/data_model.py +++ b/Whatsapp_Chat_Exporter/data_model.py @@ -65,7 +65,7 @@ class ChatStore(): class Message(): - def __init__(self, from_me: Union[bool,int], timestamp: int, time: Union[int,float,str], key_id: int, timezone_offset: int = 0): + def __init__(self, from_me: Union[bool,int], timestamp: int, time: Union[int,float,str], key_id: int, timezone_offset: int = 0, message_type: int = None): self.from_me = bool(from_me) self.timestamp = timestamp / 1000 if timestamp > 9999999999 else timestamp if isinstance(time, int) or isinstance(time, float): @@ -81,6 +81,7 @@ class Message(): self.sender = None self.safe = False self.mime = None + self.message_type = message_type # Extra self.reply = None self.quoted_data = None diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 0744f4f..97b0c88 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -7,7 +7,8 @@ from pathlib import Path from mimetypes import MimeTypes from markupsafe import escape as htmle from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition, slugify +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, CURRENT_TZ_OFFSET, get_chat_condition +from Whatsapp_Chat_Exporter.utility import bytes_to_readable, convert_time_unit, slugify, Device def contacts(db, data): @@ -26,8 +27,9 @@ def contacts(db, data): content = c.fetchone() -def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): +def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty): c = db.cursor() + cursor2 = db.cursor() # Get contacts c.execute( f"""SELECT count() @@ -149,7 +151,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): timestamp=ts, time=ts, # TODO: Could be bug key_id=content["ZSTANZAID"][:17], - timezone_offset=timezone_offset + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET, + message_type=content["ZMESSAGETYPE"] ) invalid = False if is_group_message and content["ZISFROMME"] == 0: @@ -189,7 +192,11 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): if content["ZMETADATA"] is not None and content["ZMETADATA"].startswith(b"\x2a\x14"): quoted = content["ZMETADATA"][2:19] message.reply = quoted.decode() - message.quoted_data = None # TODO + cursor2.execute(f"""SELECT ZTEXT + FROM ZWAMESSAGE + WHERE ZSTANZAID LIKE '{message.reply}%'""") + quoted_content = cursor2.fetchone() + message.quoted_data = quoted_content["ZTEXT"] or quoted_content if content["ZMESSAGETYPE"] == 15: # Sticker message.sticker = True @@ -226,7 +233,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): f"Processing messages...({total_row_number}/{total_row_number})", end="\r") -def media(db, data, media_folder, filter_date, filter_chat, separate_media=False): +def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=False): c = db.cursor() # Get media c.execute(f"""SELECT count() @@ -292,7 +299,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False Path(new_folder).mkdir(parents=True, exist_ok=True) new_path = os.path.join(new_folder, current_filename) shutil.copy2(file_path, new_path) - message.data = new_path + message.data = '/'.join(new_path.split("\\")[1:]) else: message.data = "The media is missing" message.mime = "media" @@ -307,7 +314,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data, media_folder, filter_date, filter_chat): +def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty): c = db.cursor() c.execute(f"""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, ZWAMEDIAITEM.ZMESSAGE, @@ -361,3 +368,73 @@ def vcard(db, data, media_folder, filter_date, filter_chat): message.meta = True message.safe = True print(f"Processing vCards...({index + 1}/{total_row_number})", end="\r") + + +def calls(db, data, timezone_offset, filter_chat): + c = db.cursor() + c.execute(f"""SELECT count() + FROM ZWACDCALLEVENT + WHERE 1=1 + {get_chat_condition(filter_chat[0], True, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")} + {get_chat_condition(filter_chat[1], False, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")}""") + total_row_number = c.fetchone()[0] + if total_row_number == 0: + return + print(f"\nProcessing calls...({total_row_number})", end="\r") + c.execute(f"""SELECT ZCALLIDSTRING, + ZGROUPCALLCREATORUSERJIDSTRING, + ZGROUPJIDSTRING, + ZDATE, + ZOUTCOME, + ZBYTESRECEIVED + ZBYTESSENT AS bytes_transferred, + ZDURATION, + ZVIDEO, + ZMISSED, + ZINCOMING + FROM ZWACDCALLEVENT + INNER JOIN ZWAAGGREGATECALLEVENT + ON ZWACDCALLEVENT.Z1CALLEVENTS = ZWAAGGREGATECALLEVENT.Z_PK + WHERE 1=1 + {get_chat_condition(filter_chat[0], True, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")} + {get_chat_condition(filter_chat[1], False, ["ZGROUPCALLCREATORUSERJIDSTRING"], None, "ios")}""") + chat = ChatStore(Device.ANDROID, "WhatsApp Calls") + content = c.fetchone() + while content is not None: + ts = APPLE_TIME + int(content["ZDATE"]) + call = Message( + from_me=content["ZINCOMING"] == 0, + timestamp=ts, + time=ts, + key_id=content["ZCALLIDSTRING"], + timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET + ) + _jid = content["ZGROUPCALLCREATORUSERJIDSTRING"] + name = data[_jid].name if _jid in data else None + if _jid is not None and "@" in _jid: + fallback = _jid.split('@')[0] + else: + fallback = None + call.sender = name or fallback + call.meta = True + call.data = ( + f"A {'group ' if content['ZGROUPJIDSTRING'] is not None else ''}" + f"{'video' if content['ZVIDEO'] == 1 else 'voice'} " + f"call {'to' if call.from_me else 'from'} " + f"{call.sender} was " + ) + if content['ZOUTCOME'] in (1, 4): + call.data += "not answered." if call.from_me else "missed." + elif content['ZOUTCOME'] == 2: + call.data += "failed." + elif content['ZOUTCOME'] == 0: + call_time = convert_time_unit(int(content['ZDURATION'])) + call_bytes = bytes_to_readable(content['bytes_transferred']) + call.data += ( + f"initiated and lasted for {call_time} " + f"with {call_bytes} data transferred." + ) + else: + call.data += "in an unknown state." + chat.add_message(call.key_id, call) + content = c.fetchone() + data["000000000000000"] = chat \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/ios_media_handler.py b/Whatsapp_Chat_Exporter/ios_media_handler.py index c4360af..dc817b6 100644 --- a/Whatsapp_Chat_Exporter/ios_media_handler.py +++ b/Whatsapp_Chat_Exporter/ios_media_handler.py @@ -35,6 +35,11 @@ def extract_encrypted(base_dir, password, identifiers, decrypt_chunk_size): domain_like=identifiers.DOMAIN, output_filename=identifiers.CONTACT ) + backup.extract_file( + relative_path=RelativePath.WHATSAPP_CALLS, + domain_like=identifiers.DOMAIN, + output_filename=identifiers.CALL + ) except ValueError: print("Failed to decrypt backup: incorrect password?") exit(7) @@ -87,6 +92,7 @@ def extract_media(base_dir, identifiers, decrypt_chunk_size): else: wts_db = os.path.join(base_dir, identifiers.MESSAGE[:2], identifiers.MESSAGE) contact_db = os.path.join(base_dir, identifiers.CONTACT[:2], identifiers.CONTACT) + call_db = os.path.join(base_dir, identifiers.CALL[:2], identifiers.CALL) if not os.path.isfile(wts_db): if identifiers is WhatsAppIdentifier: print("WhatsApp database not found.") @@ -99,6 +105,10 @@ def extract_media(base_dir, identifiers, decrypt_chunk_size): print("Contact database not found. Skipping...") else: shutil.copyfile(contact_db, identifiers.CONTACT) + if not os.path.isfile(call_db): + print("Call database not found. Skipping...") + else: + shutil.copyfile(call_db, identifiers.CALL) _wts_id = identifiers.DOMAIN with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: manifest.row_factory = sqlite3.Row diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 9f8c45c..259c318 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -13,6 +13,7 @@ try: from enum import StrEnum, IntEnum except ImportError: # < Python 3.11 + # This should be removed when the support for Python 3.10 ends. from enum import Enum class StrEnum(str, Enum): pass @@ -22,6 +23,7 @@ except ImportError: MAX_SIZE = 4 * 1024 * 1024 # Default 4MB ROW_SIZE = 0x3D0 +CURRENT_TZ_OFFSET = datetime.now().astimezone().utcoffset().seconds / 3600 def convert_time_unit(time_second: int): @@ -130,13 +132,18 @@ def rendering( msgs, contact, w3css, - next, chat, + headline, + next=False, + previous=False ): if chat.their_avatar_thumb is None and chat.their_avatar is not None: their_avatar_thumb = chat.their_avatar else: their_avatar_thumb = chat.their_avatar_thumb + if "??" not in headline: + raise ValueError("Headline must contain '??' to replace with name") + headline = headline.replace("??", name) with open(output_file_name, "w", encoding="utf-8") as f: f.write( template.render( @@ -147,8 +154,10 @@ def rendering( their_avatar_thumb=their_avatar_thumb, w3css=w3css, next=next, + previous=previous, status=chat.status, - media_base=chat.media_base + media_base=chat.media_base, + headline=headline ) ) @@ -218,6 +227,10 @@ def get_file_name(contact: str, chat: ChatStore): return sanitize_filename(file_name), name +def get_cond_for_empty(enable, jid_field: str, broadcast_field: str): + return f"AND (chat.hidden=0 OR {jid_field}='status@broadcast' OR {broadcast_field}>0)" if enable else "" + + def get_chat_condition(filter, include, columns, jid=None, platform=None): if filter is not None: conditions = [] @@ -243,12 +256,6 @@ def get_chat_condition(filter, include, columns, jid=None, platform=None): else: return "" -def _is_message_empty(message): - return (message.data is None or message.data == "") and not message.media - -def chat_is_empty(chat: ChatStore): - return len(chat.messages) == 0 or all(_is_message_empty(message) for message in chat.messages.values()) - # Android Specific CRYPT14_OFFSETS = ( @@ -257,6 +264,7 @@ CRYPT14_OFFSETS = ( {"iv": 66, "db": 99}, {"iv": 67, "db": 193}, {"iv": 67, "db": 194}, + {"iv": 67, "db": 158}, ) @@ -373,10 +381,10 @@ def get_status_location(output_folder, offline_static): w3css = os.path.join(offline_static, "w3.css") -def setup_template(template, no_avatar): - if template is None: +def setup_template(template, no_avatar, experimental=False): + if template is None or experimental: template_dir = os.path.dirname(__file__) - template_file = "whatsapp.html" + template_file = "whatsapp.html" if not experimental else template else: template_dir = os.path.dirname(template) template_file = os.path.basename(template) @@ -390,7 +398,7 @@ def setup_template(template, no_avatar): return template_env.get_template(template_file) # iOS Specific -APPLE_TIME = datetime.timestamp(datetime(2001, 1, 1)) +APPLE_TIME = 978307200 def slugify(value, allow_unicode=False): @@ -413,6 +421,7 @@ def slugify(value, allow_unicode=False): class WhatsAppIdentifier(StrEnum): MESSAGE = "7c7fba66680ef796b916b067077cc246adacf01d" CONTACT = "b8548dc30aa1030df0ce18ef08b882cf7ab5212f" + CALL = "1b432994e958845fffe8e2f190f26d1511534088" DOMAIN = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" diff --git a/Whatsapp_Chat_Exporter/whatsapp.html b/Whatsapp_Chat_Exporter/whatsapp.html index 20caf30..ae83b95 100644 --- a/Whatsapp_Chat_Exporter/whatsapp.html +++ b/Whatsapp_Chat_Exporter/whatsapp.html @@ -20,7 +20,6 @@ } footer { border-top: 2px solid #e3e6e7; - font-size: 2em; padding: 20px 0 20px 0; } article { @@ -91,7 +90,7 @@
- Chat history with {{ name }} + {{ headline }} {% if status is not none %}
{{ status }} @@ -121,7 +120,7 @@ {% if msg.reply is not none %}
Replying to - + {% if msg.quoted_data is not none %} "{{msg.quoted_data}}" {% else %} @@ -156,8 +155,8 @@ {% elif "video/" in msg.mime %} -
@@ -211,7 +210,7 @@ {% if msg.reply is not none %}
Replying to - + {% if msg.quoted_data is not none %} "{{msg.quoted_data}}" {% else %} @@ -246,8 +245,8 @@ {% elif "video/" in msg.mime %} - + + \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/whatsapp_new.html b/Whatsapp_Chat_Exporter/whatsapp_new.html new file mode 100644 index 0000000..7502a7b --- /dev/null +++ b/Whatsapp_Chat_Exporter/whatsapp_new.html @@ -0,0 +1,424 @@ + + + + Whatsapp - {{ name }} + + + + + + + + +
+
+
+
+ {% if not no_avatar %} +
+ {% if their_avatar is not none %} + + {% else %} + + {% endif %} +
+ {% endif %} +
+

{{ headline }}

+ {% if status is not none %}

{{ status }}

{% endif %} +
+
+
+ + + {% if previous %} + + + + + + {% endif %} + {% if next %} + + + + + + {% endif %} +
+ +
+ + +
+
+
+
+
+ + {% set last = {'last': 946688461.001} %} + {% for msg in msgs -%} + {% if determine_day(last.last, msg.timestamp) is not none %} +
+
+ {{ determine_day(last.last, msg.timestamp) }} +
+
+ {% if last.update({'last': msg.timestamp}) %}{% endif %} + {% endif %} + + {% if msg.from_me == true %} +
+
+ {% if msg.reply is not none %} + +
+

Replying to

+

+ {% if msg.quoted_data is not none %} + "{{msg.quoted_data}}" + {% else %} + this message + {% endif %} +

+
+ + {% endif %} +

+ {% if msg.meta == true or msg.media == false and msg.data is none %} +

+
+ {% if msg.safe %} + {{ msg.data | safe or 'Not supported WhatsApp internal message' }} + {% else %} + {{ msg.data or 'Not supported WhatsApp internal message' }} + {% endif %} +
+
+ {% if msg.caption is not none %} +

{{ msg.caption | urlize(none, true, '_blank') }}

+ {% endif %} + {% else %} + {% if msg.media == false %} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} + {% else %} + {% if "image/" in msg.mime %} + + + + {% elif "audio/" in msg.mime %} + + {% elif "video/" in msg.mime %} + + {% elif "/" in msg.mime %} + The file cannot be displayed here, however it should be located at here + {% else %} + {% filter escape %}{{ msg.data }}{% endfilter %} + {% endif %} + {% if msg.caption is not none %} + {{ msg.caption | urlize(none, true, '_blank') }} + {% endif %} + {% endif %} + {% endif %} +

+

{{ msg.time }}

+
+
+ {% else %} +
+
+ {% if msg.reply is not none %} + +
+

Replying to

+

+ {% if msg.quoted_data is not none %} + {{msg.quoted_data}} + {% else %} + this message + {% endif %} +

+
+ + {% endif %} +

+ {% if msg.meta == true or msg.media == false and msg.data is none %} +

+
+ {% if msg.safe %} + {{ msg.data | safe or 'Not supported WhatsApp internal message' }} + {% else %} + {{ msg.data or 'Not supported WhatsApp internal message' }} + {% endif %} +
+
+ {% if msg.caption is not none %} +

{{ msg.caption | urlize(none, true, '_blank') }}

+ {% endif %} + {% else %} + {% if msg.media == false %} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} + {% else %} + {% if "image/" in msg.mime %} + + + + {% elif "audio/" in msg.mime %} + + {% elif "video/" in msg.mime %} + + {% elif "/" in msg.mime %} + The file cannot be displayed here, however it should be located at here + {% else %} + {% filter escape %}{{ msg.data }}{% endfilter %} + {% endif %} + {% if msg.caption is not none %} + {{ msg.caption | urlize(none, true, '_blank') }} + {% endif %} + {% endif %} + {% endif %} +

+
+ + {% if msg.sender is not none %} + {{ msg.sender }} + {% endif %} + + + {{ msg.time }} +
+
+
+ {% endif %} + {% endfor %} +
+
+

+ {% if not next %} + End of History + {% endif %} +

+
+ Portions of this page are reproduced from work created and shared by Google and used according to terms described in the Apache 2.0 License. +
+
+
+ + + + + \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e442fe0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "whatsapp-chat-exporter" +version = "0.10.5" +description = "A Whatsapp database parser that provides history of your Whatsapp conversations in HTML and JSON. Android, iOS, iPadOS, Crypt12, Crypt14, Crypt15 supported." +readme = "README.md" +authors = [ + { name = "KnugiHK", email = "hello@knugi.com" } +] +license = { text = "MIT" } +keywords = [ + "android", "ios", "parsing", "history", "iphone", "message", "crypt15", + "customizable", "whatsapp", "android-backup", "messages", "crypt14", + "crypt12", "whatsapp-chat-exporter", "whatsapp-export", "iphone-backup", + "whatsapp-database", "whatsapp-database-parser", "whatsapp-conversations" +] +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "Topic :: Communications :: Chat", + "Topic :: Utilities", + "Topic :: Database" +] +requires-python = ">=3.9" +dependencies = [ + "jinja2", + "bleach" +] + +[project.optional-dependencies] +android_backup = ["pycryptodome", "javaobj-py3"] +crypt12 = ["pycryptodome"] +crypt14 = ["pycryptodome"] +crypt15 = ["pycryptodome", "javaobj-py3"] +all = ["pycryptodome", "javaobj-py3", "vobject"] +everything = ["pycryptodome", "javaobj-py3", "vobject"] +backup = ["pycryptodome", "javaobj-py3"] +vcards = ["vobject", "pycryptodome", "javaobj-py3"] + +[project.scripts] +wtsexporter = "Whatsapp_Chat_Exporter.__main__:main" +waexporter = "Whatsapp_Chat_Exporter.__main__:main" +whatsapp-chat-exporter = "Whatsapp_Chat_Exporter.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["Whatsapp_Chat_Exporter"] + +[tool.setuptools.package-data] +template = ["whatsapp.html"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 5fa5484..0000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -import setuptools -from re import search - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open("Whatsapp_Chat_Exporter/__init__.py", encoding="utf8") as f: - version = search(r'__version__ = "(.*?)"', f.read()).group(1) - -setuptools.setup( - name="whatsapp-chat-exporter", - version=version, - author="KnugiHK", - author_email="hello@knugi.com", - description=("A Whatsapp database parser that will give you the " - "history of your Whatsapp conversations in HTML and JSON. " - "Android, iOS, iPadOS, Crypt12, Crypt14, Crypt15 supported."), - long_description=long_description, - long_description_content_type="text/markdown", - license="MIT", - keywords=[ - "android", "ios", "parsing", "history", "iphone", "message", "crypt15", - "customizable", "whatsapp", "android-backup", "messages", "crypt14", - "crypt12", "whatsapp-chat-exporter", "whatsapp-export", "iphone-backup", - "whatsapp-database", "whatsapp-database-parser", "whatsapp-conversations" - ], - platforms=["any"], - url="https://github.com/KnugiHK/Whatsapp-Chat-Exporter", - packages=setuptools.find_packages(), - package_data={ - '': ['whatsapp.html'] - }, - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Topic :: Communications :: Chat", - "Topic :: Utilities", - "Topic :: Database" - ], - python_requires='>=3.8', - install_requires=[ - 'jinja2', - 'bleach' - ], - extras_require={ - 'android_backup': ["pycryptodome", "javaobj-py3"], - 'crypt12': ["pycryptodome"], - 'crypt14': ["pycryptodome"], - 'crypt15': ["pycryptodome", "javaobj-py3"], - 'all': ["pycryptodome", "javaobj-py3", "vobject"], - 'everything': ["pycryptodome", "javaobj-py3", "vobject"], - 'backup': ["pycryptodome", "javaobj-py3"], - 'vcards': ["vobject", "pycryptodome", "javaobj-py3"], - }, - entry_points={ - "console_scripts": [ - "wtsexporter = Whatsapp_Chat_Exporter.__main__:main", - "waexporter = Whatsapp_Chat_Exporter.__main__:main", - "whatsapp-chat-exporter = Whatsapp_Chat_Exporter.__main__:main" - ] - } -)