From bd2f063cc02e7eb742b851cc879f359b9abb5a04 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 12 Aug 2023 15:51:05 +0800 Subject: [PATCH 01/24] Bug fix on calls without chat #58 --- Whatsapp_Chat_Exporter/extract.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 49b6c25..6441eff 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -593,10 +593,13 @@ def calls(db, data): video_call, duration, call_result, - bytes_transferred + bytes_transferred, + chat.subject as chat_subject FROM call_log INNER JOIN jid - ON call_log.jid_row_id = jid._id""" + ON call_log.jid_row_id = jid._id + LEFT JOIN chat + ON call_log.jid_row_id = chat.jid_row_id""" ) chat = ChatStore(Device.ANDROID, "WhatsApp Calls") content = c.fetchone() @@ -608,16 +611,17 @@ def calls(db, data): key_id=content["call_id"], ) _jid = content["raw_string"] - if _jid in data: - name = data[_jid].name - fallback = _jid.split('@')[0] if "@" in _jid else None - call.sender = name or fallback - + name = data[_jid].name if _jid in data else content["chat_subject"] or None + if _jid is not None and "@" in _jid: + fallback = _jid.split('@')[0] + else: + fallback = None + call.sender = name or fallback call.meta = True call.data = ( f"A {'video' if content['video_call'] else 'voice'} " f"call {'to' if call.from_me else 'from'} " - f"{name or fallback} was " + f"{call.sender} was " ) if content['call_result'] == 2: call.data += "not answered." if call.from_me else "missed." From 326b99d8600e429a93b8e1543b35902b4a738c23 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 15 Aug 2023 00:32:07 -0400 Subject: [PATCH 02/24] feat(whatsapp_business)!: extraction script for smb Duplicated original extract_iphone_media to include smb specfic fileIDs. --- .../extract_iphone_media_smb.py | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 Whatsapp_Chat_Exporter/extract_iphone_media_smb.py diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py b/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py new file mode 100644 index 0000000..0f9511d --- /dev/null +++ b/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py @@ -0,0 +1,126 @@ +#!/usr/bin/python3 + +import shutil +import sqlite3 +import os +import time +import getpass +import threading +try: + from iphone_backup_decrypt import EncryptedBackup, RelativePath + from iphone_backup_decrypt import FailedToDecryptError, Domain +except ModuleNotFoundError: + support_encrypted = False +else: + support_encrypted = True + + +def extract_encrypted(base_dir, password): + backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) + print("Decrypting WhatsApp database...") + try: + backup.extract_file(relative_path=RelativePath.WHATSAPP_MESSAGES, + output_filename="724bd3b98b18518b455a87c1f3ac3a0d189c4466") + backup.extract_file(relative_path=RelativePath.WHATSAPP_CONTACTS, + output_filename="d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") + except FailedToDecryptError: + print("Failed to decrypt backup: incorrect password?") + exit() + extract_thread = threading.Thread( + target=backup.extract_files_by_domain, + args=(Domain.WHATSAPP, Domain.WHATSAPP) + ) + extract_thread.daemon = True + extract_thread.start() + dot = 0 + while extract_thread.is_alive(): + print(f"Decrypting and extracting files{'.' * dot}{' ' * (3 - dot)}", end="\r") + if dot < 3: + dot += 1 + time.sleep(0.5) + else: + dot = 0 + time.sleep(0.4) + print(f"All required files decrypted and extracted.", end="\n") + extract_thread.handled = True + return backup + + +def is_encrypted(base_dir): + with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as f: + c = f.cursor() + try: + c.execute("""SELECT count() + FROM Files + """) + except sqlite3.OperationalError as e: + raise e # These error cannot be used to determine if the backup is encrypted + except sqlite3.DatabaseError: + return True + else: + return False + + +def extract_media(base_dir): + if is_encrypted(base_dir): + if not support_encrypted: + print("You don't have the dependencies to handle encrypted backup.") + print("Read more on how to deal with encrypted backup:") + print("https://github.com/KnugiHK/Whatsapp-Chat-Exporter/blob/main/README.md#usage") + return False + print("Encryption detected on the backup!") + password = getpass.getpass("Enter the password for the backup:") + extract_encrypted(base_dir, password) + else: + wts_db = os.path.join(base_dir, "72/724bd3b98b18518b455a87c1f3ac3a0d189c4466") + contact_db = os.path.join(base_dir, "d7/d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") + if not os.path.isfile(wts_db): + print("WhatsApp database not found.") + exit() + else: + shutil.copyfile(wts_db, "724bd3b98b18518b455a87c1f3ac3a0d189c4466") + if not os.path.isfile(contact_db): + print("Contact database not found.") + exit() + else: + shutil.copyfile(contact_db, "d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") + _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: + manifest.row_factory = sqlite3.Row + c = manifest.cursor() + c.execute( + f"""SELECT count() + FROM Files + WHERE domain = '{_wts_id}'""" + ) + total_row_number = c.fetchone()[0] + print(f"Extracting WhatsApp files...(0/{total_row_number})", end="\r") + c.execute(f"""SELECT fileID, + relativePath, + flags, + ROW_NUMBER() OVER(ORDER BY relativePath) AS _index + FROM Files + WHERE domain = '{_wts_id}' + ORDER BY relativePath""") + if not os.path.isdir(_wts_id): + os.mkdir(_wts_id) + row = c.fetchone() + while row is not None: + if row["relativePath"] == "": + row = c.fetchone() + continue + destination = os.path.join(_wts_id, row["relativePath"]) + hashes = row["fileID"] + folder = hashes[:2] + flags = row["flags"] + if flags == 2: + try: + os.mkdir(destination) + except FileExistsError: + pass + elif flags == 1: + shutil.copyfile(os.path.join(base_dir, folder, hashes), destination) + if row["_index"] % 100 == 0: + print(f"Extracting WhatsApp files...({row['_index']}/{total_row_number})", end="\r") + row = c.fetchone() + print(f"Extracting WhatsApp files...({total_row_number}/{total_row_number})", end="\n") From d8b434e169b3ecdef42dab1fcc8719c42e0bc2eb Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 15 Aug 2023 00:35:32 -0400 Subject: [PATCH 03/24] feat(whatsapp_business)!: args and default values for smb Added corresponding logic for whatsapp business and default values. --- Whatsapp_Chat_Exporter/__main__.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 14e1f6d..370b929 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -7,7 +7,7 @@ import json import string import glob from Whatsapp_Chat_Exporter import extract_exported, extract_iphone -from Whatsapp_Chat_Exporter import extract, extract_iphone_media +from Whatsapp_Chat_Exporter import extract, extract_iphone_media, extract_iphone_media_smb from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import Crypt, check_update, import_from_json from argparse import ArgumentParser, SUPPRESS @@ -177,6 +177,13 @@ def main(): action='store_true', help="Import JSON file and convert to HTML output" ) + parser.add_argument( + "--smb", + dest="smb", + default=False, + action='store_true', + help="Use Whatsapp Business default files (iphone only)" + ) args = parser.parse_args() # Check for updates @@ -265,14 +272,23 @@ def main(): vcard = extract_iphone.vcard create_html = extract.create_html if args.media is None: - args.media = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + if args.smb: + args.media = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" + else: + args.media = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" if args.backup is not None: if not os.path.isdir(args.media): - extract_iphone_media.extract_media(args.backup) + if args.smb: + extract_iphone_media_smb.extract_media(args.backup) + else: + extract_iphone_media.extract_media(args.backup) else: print("WhatsApp directory already exists, skipping WhatsApp file extraction.") if args.db is None: - msg_db = "7c7fba66680ef796b916b067077cc246adacf01d" + if args.smb: + msg_db = "724bd3b98b18518b455a87c1f3ac3a0d189c4466" + else: + msg_db = "7c7fba66680ef796b916b067077cc246adacf01d" else: msg_db = args.db if args.wa is None: From 269a59c1e21e695ef94dfc36281aa01b0d8583b7 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 15 Aug 2023 00:36:15 -0400 Subject: [PATCH 04/24] feat(whatsapp_business): made vcard path dynamic --- Whatsapp_Chat_Exporter/__main__.py | 2 +- Whatsapp_Chat_Exporter/extract_iphone.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 370b929..66deabe 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -306,7 +306,7 @@ def main(): db.row_factory = sqlite3.Row messages(db, data, args.media) media(db, data, args.media) - vcard(db, data) + vcard(db, data, args.media) if args.android: extract.calls(db, data) if not args.no_html: diff --git a/Whatsapp_Chat_Exporter/extract_iphone.py b/Whatsapp_Chat_Exporter/extract_iphone.py index 822ad24..dadd695 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone.py +++ b/Whatsapp_Chat_Exporter/extract_iphone.py @@ -244,7 +244,7 @@ def media(db, data, media_folder): f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data): +def vcard(db, data, media_folder): c = db.cursor() c.execute("""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, ZWAMEDIAITEM.ZMESSAGE, @@ -260,13 +260,13 @@ def vcard(db, data): contents = c.fetchall() total_row_number = len(contents) print(f"\nProcessing vCards...(0/{total_row_number})", end="\r") - base = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared/Message/vCards" - if not os.path.isdir(base): - Path(base).mkdir(parents=True, exist_ok=True) + path = f'{media_folder}/Message/vCards' + if not os.path.isdir(path): + Path(path).mkdir(parents=True, exist_ok=True) for index, content in enumerate(contents): file_name = "".join(x for x in content["ZVCARDNAME"] if x.isalnum()) file_name = file_name.encode('utf-8')[:230].decode('utf-8', 'ignore') - file_path = os.path.join(base, f"{file_name}.vcf") + file_path = os.path.join(path, f"{file_name}.vcf") if not os.path.isfile(file_path): with open(file_path, "w", encoding="utf-8") as f: f.write(content["ZVCARDSTRING"]) From f4888949429c9411728a9f40a91b5d7a9bc9b6bc Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 15 Aug 2023 00:46:54 -0400 Subject: [PATCH 05/24] fix(whatsapp_business): missing domain ref --- Whatsapp_Chat_Exporter/extract_iphone_media_smb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py b/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py index 0f9511d..459344a 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py @@ -84,7 +84,7 @@ def extract_media(base_dir): exit() else: shutil.copyfile(contact_db, "d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") - _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: manifest.row_factory = sqlite3.Row c = manifest.cursor() From ee4e95c75f17eff48abafda747d920a324ecd9f6 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:22:08 +0800 Subject: [PATCH 06/24] Bug fix on the possible breakage mentioned in #60 --- Whatsapp_Chat_Exporter/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 6441eff..9dd552d 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -528,7 +528,7 @@ def media(db, data, media_folder): f"Processing media...({total_row_number}/{total_row_number})", end="\r") -def vcard(db, data): +def vcard(db, data, media_folder): c = db.cursor() try: c.execute("""SELECT message_row_id, From a5cb46e095c70aeaa0a6aa32955914965650a76f Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:22:43 +0800 Subject: [PATCH 07/24] Also make vcard path dynamic in Android --- Whatsapp_Chat_Exporter/extract.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 9dd552d..2b5fb90 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -558,14 +558,14 @@ def vcard(db, data, media_folder): rows = c.fetchall() total_row_number = len(rows) print(f"\nProcessing vCards...(0/{total_row_number})", end="\r") - base = "WhatsApp/vCards" - if not os.path.isdir(base): - Path(base).mkdir(parents=True, exist_ok=True) + path = f"{media_folder}/vCards" + if not os.path.isdir(path): + Path(path).mkdir(parents=True, exist_ok=True) for index, row in enumerate(rows): media_name = row["media_name"] if row["media_name"] is not None else "" file_name = "".join(x for x in media_name if x.isalnum()) file_name = file_name.encode('utf-8')[:230].decode('utf-8', 'ignore') - file_path = os.path.join(base, f"{file_name}.vcf") + file_path = os.path.join(path, f"{file_name}.vcf") if not os.path.isfile(file_path): with open(file_path, "w", encoding="utf-8") as f: f.write(row["vcard"]) From 448ba892cc1b650356fd4881f319dd665ea5d8ad Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 15 Aug 2023 17:33:56 +0800 Subject: [PATCH 08/24] Change the option from --smb to --business --- Whatsapp_Chat_Exporter/__main__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 66deabe..45f708a 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -178,11 +178,11 @@ def main(): help="Import JSON file and convert to HTML output" ) parser.add_argument( - "--smb", - dest="smb", + "--business", + dest="business", default=False, action='store_true', - help="Use Whatsapp Business default files (iphone only)" + help="Use Whatsapp Business default files (iOS only)" ) args = parser.parse_args() @@ -206,6 +206,9 @@ def main(): elif args.import_json and not os.path.isfile(args.json): print("JSON file not found.") exit(1) + if args.android and args.business: + print("WhatsApp Business is only available on iOS for now.") + exit(1) data = {} From 2944d00ca29296a67a533e5744e9d7794b69995d Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 15 Aug 2023 17:40:20 +0800 Subject: [PATCH 09/24] Refactoring so that no file needs to be introduced --- Whatsapp_Chat_Exporter/__main__.py | 21 ++- .../extract_iphone_media.py | 20 +-- .../extract_iphone_media_smb.py | 126 ------------------ Whatsapp_Chat_Exporter/utility.py | 12 ++ 4 files changed, 32 insertions(+), 147 deletions(-) delete mode 100644 Whatsapp_Chat_Exporter/extract_iphone_media_smb.py diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 45f708a..c20d8bd 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -7,7 +7,7 @@ import json import string import glob from Whatsapp_Chat_Exporter import extract_exported, extract_iphone -from Whatsapp_Chat_Exporter import extract, extract_iphone_media, extract_iphone_media_smb +from Whatsapp_Chat_Exporter import extract, extract_iphone_media from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import Crypt, check_update, import_from_json from argparse import ArgumentParser, SUPPRESS @@ -274,24 +274,19 @@ def main(): media = extract_iphone.media vcard = extract_iphone.vcard create_html = extract.create_html + if args.business: + from Whatsapp_Chat_Exporter.utility import WhatsAppBusinessIdentifier as identifiers + else: + from Whatsapp_Chat_Exporter.utility import WhatsAppIdentifier as identifiers if args.media is None: - if args.smb: - args.media = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" - else: - args.media = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + args.media = identifiers.DOMAIN if args.backup is not None: if not os.path.isdir(args.media): - if args.smb: - extract_iphone_media_smb.extract_media(args.backup) - else: - extract_iphone_media.extract_media(args.backup) + extract_iphone_media.extract_media(args.backup, identifiers) else: print("WhatsApp directory already exists, skipping WhatsApp file extraction.") if args.db is None: - if args.smb: - msg_db = "724bd3b98b18518b455a87c1f3ac3a0d189c4466" - else: - msg_db = "7c7fba66680ef796b916b067077cc246adacf01d" + msg_db = identifiers.MESSAGE else: msg_db = args.db if args.wa is None: diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index b293ea7..250d4d6 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -6,6 +6,7 @@ import os import time import getpass import threading +from Whatsapp_Chat_Exporter.utility import WhatsAppIdentifier try: from iphone_backup_decrypt import EncryptedBackup, RelativePath from iphone_backup_decrypt import FailedToDecryptError, Domain @@ -15,7 +16,7 @@ else: support_encrypted = True -def extract_encrypted(base_dir, password): +def extract_encrypted(base_dir, password, identifiers): backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) print("Decrypting WhatsApp database...") try: @@ -61,7 +62,7 @@ def is_encrypted(base_dir): return False -def extract_media(base_dir): +def extract_media(base_dir, identifiers): if is_encrypted(base_dir): if not support_encrypted: print("You don't have the dependencies to handle encrypted backup.") @@ -70,20 +71,23 @@ def extract_media(base_dir): return False print("Encryption detected on the backup!") password = getpass.getpass("Enter the password for the backup:") - extract_encrypted(base_dir, password) + extract_encrypted(base_dir, password, identifiers) else: - wts_db = os.path.join(base_dir, "7c/7c7fba66680ef796b916b067077cc246adacf01d") - contact_db = os.path.join(base_dir, "b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f") + wts_db = os.path.join(base_dir, identifiers.MESSAGE[:2], identifiers.MESSAGE) + contact_db = os.path.join(base_dir, identifiers.CONTACT[:2], identifiers.CONTACT) if not os.path.isfile(wts_db): - print("WhatsApp database not found.") + if identifiers is WhatsAppIdentifier: + print("WhatsApp database not found.") + else: + print("WhatsApp Business database not found.") exit() else: - shutil.copyfile(wts_db, "7c7fba66680ef796b916b067077cc246adacf01d") + shutil.copyfile(wts_db, identifiers.MESSAGE) if not os.path.isfile(contact_db): print("Contact database not found.") exit() else: - shutil.copyfile(contact_db, "b8548dc30aa1030df0ce18ef08b882cf7ab5212f") + shutil.copyfile(contact_db, identifiers.CONTACT) _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: manifest.row_factory = sqlite3.Row diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py b/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py deleted file mode 100644 index 459344a..0000000 --- a/Whatsapp_Chat_Exporter/extract_iphone_media_smb.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/python3 - -import shutil -import sqlite3 -import os -import time -import getpass -import threading -try: - from iphone_backup_decrypt import EncryptedBackup, RelativePath - from iphone_backup_decrypt import FailedToDecryptError, Domain -except ModuleNotFoundError: - support_encrypted = False -else: - support_encrypted = True - - -def extract_encrypted(base_dir, password): - backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) - print("Decrypting WhatsApp database...") - try: - backup.extract_file(relative_path=RelativePath.WHATSAPP_MESSAGES, - output_filename="724bd3b98b18518b455a87c1f3ac3a0d189c4466") - backup.extract_file(relative_path=RelativePath.WHATSAPP_CONTACTS, - output_filename="d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") - except FailedToDecryptError: - print("Failed to decrypt backup: incorrect password?") - exit() - extract_thread = threading.Thread( - target=backup.extract_files_by_domain, - args=(Domain.WHATSAPP, Domain.WHATSAPP) - ) - extract_thread.daemon = True - extract_thread.start() - dot = 0 - while extract_thread.is_alive(): - print(f"Decrypting and extracting files{'.' * dot}{' ' * (3 - dot)}", end="\r") - if dot < 3: - dot += 1 - time.sleep(0.5) - else: - dot = 0 - time.sleep(0.4) - print(f"All required files decrypted and extracted.", end="\n") - extract_thread.handled = True - return backup - - -def is_encrypted(base_dir): - with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as f: - c = f.cursor() - try: - c.execute("""SELECT count() - FROM Files - """) - except sqlite3.OperationalError as e: - raise e # These error cannot be used to determine if the backup is encrypted - except sqlite3.DatabaseError: - return True - else: - return False - - -def extract_media(base_dir): - if is_encrypted(base_dir): - if not support_encrypted: - print("You don't have the dependencies to handle encrypted backup.") - print("Read more on how to deal with encrypted backup:") - print("https://github.com/KnugiHK/Whatsapp-Chat-Exporter/blob/main/README.md#usage") - return False - print("Encryption detected on the backup!") - password = getpass.getpass("Enter the password for the backup:") - extract_encrypted(base_dir, password) - else: - wts_db = os.path.join(base_dir, "72/724bd3b98b18518b455a87c1f3ac3a0d189c4466") - contact_db = os.path.join(base_dir, "d7/d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") - if not os.path.isfile(wts_db): - print("WhatsApp database not found.") - exit() - else: - shutil.copyfile(wts_db, "724bd3b98b18518b455a87c1f3ac3a0d189c4466") - if not os.path.isfile(contact_db): - print("Contact database not found.") - exit() - else: - shutil.copyfile(contact_db, "d7246a707f51ddf8b17ee2dddabd9e0a4da5c552") - _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" - with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: - manifest.row_factory = sqlite3.Row - c = manifest.cursor() - c.execute( - f"""SELECT count() - FROM Files - WHERE domain = '{_wts_id}'""" - ) - total_row_number = c.fetchone()[0] - print(f"Extracting WhatsApp files...(0/{total_row_number})", end="\r") - c.execute(f"""SELECT fileID, - relativePath, - flags, - ROW_NUMBER() OVER(ORDER BY relativePath) AS _index - FROM Files - WHERE domain = '{_wts_id}' - ORDER BY relativePath""") - if not os.path.isdir(_wts_id): - os.mkdir(_wts_id) - row = c.fetchone() - while row is not None: - if row["relativePath"] == "": - row = c.fetchone() - continue - destination = os.path.join(_wts_id, row["relativePath"]) - hashes = row["fileID"] - folder = hashes[:2] - flags = row["flags"] - if flags == 2: - try: - os.mkdir(destination) - except FileExistsError: - pass - elif flags == 1: - shutil.copyfile(os.path.join(base_dir, folder, hashes), destination) - if row["_index"] % 100 == 0: - print(f"Extracting WhatsApp files...({row['_index']}/{total_row_number})", end="\r") - row = c.fetchone() - print(f"Extracting WhatsApp files...({total_row_number}/{total_row_number})", end="\n") diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 62350d0..26f1df1 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -290,3 +290,15 @@ def setup_template(template, no_avatar): # iOS Specific APPLE_TIME = datetime.timestamp(datetime(2001, 1, 1)) + + +class WhatsAppIdentifier(StrEnum): + MESSAGE = "7c7fba66680ef796b916b067077cc246adacf01d" + CONTACT = "b8548dc30aa1030df0ce18ef08b882cf7ab5212f" + DOMAIN = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + + +class WhatsAppBusinessIdentifier(StrEnum): + MESSAGE = "724bd3b98b18518b455a87c1f3ac3a0d189c4466" + CONTACT = "d7246a707f51ddf8b17ee2dddabd9e0a4da5c552" + DOMAIN = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" From b9f123fbea69d65bdacba019e50e4715641146e8 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Thu, 17 Aug 2023 19:59:11 -0400 Subject: [PATCH 10/24] fix: use corresponding identifier --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index 250d4d6..468e694 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -88,7 +88,7 @@ def extract_media(base_dir, identifiers): exit() else: shutil.copyfile(contact_db, identifiers.CONTACT) - _wts_id = "AppDomainGroup-group.net.whatsapp.WhatsApp.shared" + _wts_id = identifiers.DOMAIN with sqlite3.connect(os.path.join(base_dir, "Manifest.db")) as manifest: manifest.row_factory = sqlite3.Row c = manifest.cursor() From dbd1802dd63e0b0ac1210d1bc6d9fc2f4899cf49 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Fri, 18 Aug 2023 01:51:13 -0400 Subject: [PATCH 11/24] feat: use dynamic domains when encrypted --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index 468e694..f2ef72c 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -9,7 +9,7 @@ import threading from Whatsapp_Chat_Exporter.utility import WhatsAppIdentifier try: from iphone_backup_decrypt import EncryptedBackup, RelativePath - from iphone_backup_decrypt import FailedToDecryptError, Domain + from iphone_backup_decrypt import FailedToDecryptError except ModuleNotFoundError: support_encrypted = False else: @@ -21,15 +21,15 @@ def extract_encrypted(base_dir, password, identifiers): print("Decrypting WhatsApp database...") try: backup.extract_file(relative_path=RelativePath.WHATSAPP_MESSAGES, - output_filename="7c7fba66680ef796b916b067077cc246adacf01d") + output_filename=identifiers.MESSAGE) backup.extract_file(relative_path=RelativePath.WHATSAPP_CONTACTS, - output_filename="b8548dc30aa1030df0ce18ef08b882cf7ab5212f") + output_filename=identifiers.CONTACT) except FailedToDecryptError: print("Failed to decrypt backup: incorrect password?") exit() extract_thread = threading.Thread( target=backup.extract_files_by_domain, - args=(Domain.WHATSAPP, Domain.WHATSAPP) + args=(identifiers.DOMAIN, identifiers.DOMAIN) ) extract_thread.daemon = True extract_thread.start() From a08f44e6ed45812466ce65073f4392995b7b6ad8 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:06:25 +0800 Subject: [PATCH 12/24] Fix the potential collision of Whatsapp and Whatsapp Business --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index f2ef72c..1cf4f6b 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -20,10 +20,16 @@ def extract_encrypted(base_dir, password, identifiers): backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) print("Decrypting WhatsApp database...") try: - backup.extract_file(relative_path=RelativePath.WHATSAPP_MESSAGES, - output_filename=identifiers.MESSAGE) - backup.extract_file(relative_path=RelativePath.WHATSAPP_CONTACTS, - output_filename=identifiers.CONTACT) + backup.extract_file( + relative_path=RelativePath.WHATSAPP_MESSAGES, + domain=identifiers.DOMAIN, + output_filename=identifiers.MESSAGE + ) + backup.extract_file( + relative_path=RelativePath.WHATSAPP_CONTACTS, + domain=identifiers.DOMAIN, + output_filename=identifiers.CONTACT + ) except FailedToDecryptError: print("Failed to decrypt backup: incorrect password?") exit() From decea88028e7952e8343c77ef9abe7b371c9866d Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:07:04 +0800 Subject: [PATCH 13/24] Better UI I am too lazy to switch branch. --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index 1cf4f6b..4698723 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -18,7 +18,7 @@ else: def extract_encrypted(base_dir, password, identifiers): backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) - print("Decrypting WhatsApp database...") + print("Decrypting WhatsApp database...", end="") try: backup.extract_file( relative_path=RelativePath.WHATSAPP_MESSAGES, @@ -33,6 +33,8 @@ def extract_encrypted(base_dir, password, identifiers): except FailedToDecryptError: print("Failed to decrypt backup: incorrect password?") exit() + else: + print("Done") extract_thread = threading.Thread( target=backup.extract_files_by_domain, args=(identifiers.DOMAIN, identifiers.DOMAIN) From 91ff882d1575037d5cc52f1daf0d967a2db2b38e Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 7 Oct 2023 23:50:43 +0800 Subject: [PATCH 14/24] Add option to preserve modify timestamp #65 --- README.md | 4 +- Whatsapp_Chat_Exporter/__main__.py | 9 +- Whatsapp_Chat_Exporter/bplist.py | 292 ++++++++++++++++++ .../extract_iphone_media.py | 12 +- 4 files changed, 313 insertions(+), 4 deletions(-) create mode 100644 Whatsapp_Chat_Exporter/bplist.py diff --git a/README.md b/README.md index 257b62c..d77134c 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ See [issues](https://github.com/KnugiHK/Whatsapp-Chat-Exporter/issues). # Copyright This is a MIT licensed project. -The Telegram Desktop's export is the reference for whatsapp.html in this repo +The Telegram Desktop's export is the reference for whatsapp.html in this repo. + +bplist.py was released by Vladimir "Farcaller" Pouzanov under MIT license. WhatsApp Chat Exporter is not affiliated, associated, authorized, endorsed by, or in any way officially connected with the WhatsApp LLC, or any of its subsidiaries or its affiliates. The official WhatsApp LLC website can be found at https://www.whatsapp.com/. diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index c20d8bd..01354bf 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -184,6 +184,13 @@ def main(): action='store_true', help="Use Whatsapp Business default files (iOS only)" ) + parser.add_argument( + "--preserve-timestamp", + dest="preserve_timestamp", + default=False, + action='store_true', + help="Preserve the modification timestamp of the extracted files (iOS only)" + ) args = parser.parse_args() # Check for updates @@ -282,7 +289,7 @@ def main(): args.media = identifiers.DOMAIN if args.backup is not None: if not os.path.isdir(args.media): - extract_iphone_media.extract_media(args.backup, identifiers) + extract_iphone_media.extract_media(args.backup, identifiers, args.preserve_timestamp) else: print("WhatsApp directory already exists, skipping WhatsApp file extraction.") if args.db is None: diff --git a/Whatsapp_Chat_Exporter/bplist.py b/Whatsapp_Chat_Exporter/bplist.py new file mode 100644 index 0000000..fddd964 --- /dev/null +++ b/Whatsapp_Chat_Exporter/bplist.py @@ -0,0 +1,292 @@ +################################################################################# +# Copyright (C) 2009-2011 Vladimir "Farcaller" Pouzanov # +# # +# Permission is hereby granted, free of charge, to any person obtaining a copy # +# of this software and associated documentation files (the "Software"), to deal # +# in the Software without restriction, including without limitation the rights # +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # +# copies of the Software, and to permit persons to whom the Software is # +# furnished to do so, subject to the following conditions: # +# # +# The above copyright notice and this permission notice shall be included in # +# all copies or substantial portions of the Software. # +# # +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # +# THE SOFTWARE. # +################################################################################# + +import struct +import codecs +from datetime import datetime, timedelta + +class BPListWriter(object): + def __init__(self, objects): + self.bplist = "" + self.objects = objects + + def binary(self): + '''binary -> string + + Generates bplist + ''' + self.data = 'bplist00' + + # TODO: flatten objects and count max length size + + # TODO: write objects and save offsets + + # TODO: write offsets + + # TODO: write metadata + + return self.data + + def write(self, filename): + ''' + + Writes bplist to file + ''' + if self.bplist != "": + pass + # TODO: save self.bplist to file + else: + raise Exception('BPlist not yet generated') + +class BPListReader(object): + def __init__(self, s): + self.data = s + self.objects = [] + self.resolved = {} + + def __unpackIntStruct(self, sz, s): + '''__unpackIntStruct(size, string) -> int + + Unpacks the integer of given size (1, 2 or 4 bytes) from string + ''' + if sz == 1: + ot = '!B' + elif sz == 2: + ot = '!H' + elif sz == 4: + ot = '!I' + elif sz == 8: + ot = '!Q' + else: + raise Exception('int unpack size '+str(sz)+' unsupported') + return struct.unpack(ot, s)[0] + + def __unpackInt(self, offset): + '''__unpackInt(offset) -> int + + Unpacks int field from plist at given offset + ''' + return self.__unpackIntMeta(offset)[1] + + def __unpackIntMeta(self, offset): + '''__unpackIntMeta(offset) -> (size, int) + + Unpacks int field from plist at given offset and returns its size and value + ''' + obj_header = self.data[offset] + obj_type, obj_info = (obj_header & 0xF0), (obj_header & 0x0F) + int_sz = 2**obj_info + return int_sz, self.__unpackIntStruct(int_sz, self.data[offset+1:offset+1+int_sz]) + + def __resolveIntSize(self, obj_info, offset): + '''__resolveIntSize(obj_info, offset) -> (count, offset) + + Calculates count of objref* array entries and returns count and offset to first element + ''' + if obj_info == 0x0F: + ofs, obj_count = self.__unpackIntMeta(offset+1) + objref = offset+2+ofs + else: + obj_count = obj_info + objref = offset+1 + return obj_count, objref + + def __unpackFloatStruct(self, sz, s): + '''__unpackFloatStruct(size, string) -> float + + Unpacks the float of given size (4 or 8 bytes) from string + ''' + if sz == 4: + ot = '!f' + elif sz == 8: + ot = '!d' + else: + raise Exception('float unpack size '+str(sz)+' unsupported') + return struct.unpack(ot, s)[0] + + def __unpackFloat(self, offset): + '''__unpackFloat(offset) -> float + + Unpacks float field from plist at given offset + ''' + obj_header = self.data[offset] + obj_type, obj_info = (obj_header & 0xF0), (obj_header & 0x0F) + int_sz = 2**obj_info + return int_sz, self.__unpackFloatStruct(int_sz, self.data[offset+1:offset+1+int_sz]) + + def __unpackDate(self, offset): + td = int(struct.unpack(">d", self.data[offset+1:offset+9])[0]) + return datetime(year=2001,month=1,day=1) + timedelta(seconds=td) + + def __unpackItem(self, offset): + '''__unpackItem(offset) + + Unpacks and returns an item from plist + ''' + obj_header = self.data[offset] + obj_type, obj_info = (obj_header & 0xF0), (obj_header & 0x0F) + if obj_type == 0x00: + if obj_info == 0x00: # null 0000 0000 + return None + elif obj_info == 0x08: # bool 0000 1000 // false + return False + elif obj_info == 0x09: # bool 0000 1001 // true + return True + elif obj_info == 0x0F: # fill 0000 1111 // fill byte + raise Exception("0x0F Not Implemented") # this is really pad byte, FIXME + else: + raise Exception('unpack item type '+str(obj_header)+' at '+str(offset)+ 'failed') + elif obj_type == 0x10: # int 0001 nnnn ... // # of bytes is 2^nnnn, big-endian bytes + return self.__unpackInt(offset) + elif obj_type == 0x20: # real 0010 nnnn ... // # of bytes is 2^nnnn, big-endian bytes + return self.__unpackFloat(offset) + elif obj_type == 0x30: # date 0011 0011 ... // 8 byte float follows, big-endian bytes + return self.__unpackDate(offset) + elif obj_type == 0x40: # data 0100 nnnn [int] ... // nnnn is number of bytes unless 1111 then int count follows, followed by bytes + obj_count, objref = self.__resolveIntSize(obj_info, offset) + return self.data[objref:objref+obj_count] # XXX: we return data as str + elif obj_type == 0x50: # string 0101 nnnn [int] ... // ASCII string, nnnn is # of chars, else 1111 then int count, then bytes + obj_count, objref = self.__resolveIntSize(obj_info, offset) + return self.data[objref:objref+obj_count] + elif obj_type == 0x60: # string 0110 nnnn [int] ... // Unicode string, nnnn is # of chars, else 1111 then int count, then big-endian 2-byte uint16_t + obj_count, objref = self.__resolveIntSize(obj_info, offset) + return self.data[objref:objref+obj_count*2].decode('utf-16be') + elif obj_type == 0x80: # uid 1000 nnnn ... // nnnn+1 is # of bytes + # FIXME: Accept as a string for now + obj_count, objref = self.__resolveIntSize(obj_info, offset) + return self.data[objref:objref+obj_count] + elif obj_type == 0xA0: # array 1010 nnnn [int] objref* // nnnn is count, unless '1111', then int count follows + obj_count, objref = self.__resolveIntSize(obj_info, offset) + arr = [] + for i in range(obj_count): + arr.append(self.__unpackIntStruct(self.object_ref_size, self.data[objref+i*self.object_ref_size:objref+i*self.object_ref_size+self.object_ref_size])) + return arr + elif obj_type == 0xC0: # set 1100 nnnn [int] objref* // nnnn is count, unless '1111', then int count follows + # XXX: not serializable via apple implementation + raise Exception("0xC0 Not Implemented") # FIXME: implement + elif obj_type == 0xD0: # dict 1101 nnnn [int] keyref* objref* // nnnn is count, unless '1111', then int count follows + obj_count, objref = self.__resolveIntSize(obj_info, offset) + keys = [] + for i in range(obj_count): + keys.append(self.__unpackIntStruct(self.object_ref_size, self.data[objref+i*self.object_ref_size:objref+i*self.object_ref_size+self.object_ref_size])) + values = [] + objref += obj_count*self.object_ref_size + for i in range(obj_count): + values.append(self.__unpackIntStruct(self.object_ref_size, self.data[objref+i*self.object_ref_size:objref+i*self.object_ref_size+self.object_ref_size])) + dic = {} + for i in range(obj_count): + dic[keys[i]] = values[i] + return dic + else: + raise Exception('don\'t know how to unpack obj type '+hex(obj_type)+' at '+str(offset)) + + def __resolveObject(self, idx): + try: + return self.resolved[idx] + except KeyError: + obj = self.objects[idx] + if type(obj) == list: + newArr = [] + for i in obj: + newArr.append(self.__resolveObject(i)) + self.resolved[idx] = newArr + return newArr + if type(obj) == dict: + newDic = {} + for k,v in obj.items(): + key_resolved = self.__resolveObject(k) + if isinstance(key_resolved, str): + rk = key_resolved + else: + rk = codecs.decode(key_resolved, "utf-8") + rv = self.__resolveObject(v) + newDic[rk] = rv + self.resolved[idx] = newDic + return newDic + else: + self.resolved[idx] = obj + return obj + + def parse(self): + # read header + if self.data[:8] != b'bplist00': + raise Exception('Bad magic') + + # read trailer + self.offset_size, self.object_ref_size, self.number_of_objects, self.top_object, self.table_offset = struct.unpack('!6xBB4xI4xI4xI', self.data[-32:]) + #print "** plist offset_size:",self.offset_size,"objref_size:",self.object_ref_size,"num_objs:",self.number_of_objects,"top:",self.top_object,"table_ofs:",self.table_offset + + # read offset table + self.offset_table = self.data[self.table_offset:-32] + self.offsets = [] + ot = self.offset_table + for i in range(self.number_of_objects): + offset_entry = ot[:self.offset_size] + ot = ot[self.offset_size:] + self.offsets.append(self.__unpackIntStruct(self.offset_size, offset_entry)) + #print "** plist offsets:",self.offsets + + # read object table + self.objects = [] + k = 0 + for i in self.offsets: + obj = self.__unpackItem(i) + #print "** plist unpacked",k,type(obj),obj,"at",i + k += 1 + self.objects.append(obj) + + # rebuild object tree + #for i in range(len(self.objects)): + # self.__resolveObject(i) + + # return root object + return self.__resolveObject(self.top_object) + + @classmethod + def plistWithString(cls, s): + parser = cls(s) + return parser.parse() + +# helpers for testing +def plist(obj): + from Foundation import NSPropertyListSerialization, NSPropertyListBinaryFormat_v1_0 + b = NSPropertyListSerialization.dataWithPropertyList_format_options_error_(obj, NSPropertyListBinaryFormat_v1_0, 0, None) + return str(b.bytes()) + +def unplist(s): + from Foundation import NSData, NSPropertyListSerialization + d = NSData.dataWithBytes_length_(s, len(s)) + return NSPropertyListSerialization.propertyListWithData_options_format_error_(d, 0, None, None) + +if __name__ == "__main__": + import os + import sys + import json + file_path = sys.argv[1] + + with open(file_path, "rb") as fp: + data = fp.read() + + out = BPListReader(data).parse() + + with open(file_path + ".json", "w") as fp: + json.dump(out, indent=4) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index 4698723..2fc6e5c 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -16,7 +16,7 @@ else: support_encrypted = True -def extract_encrypted(base_dir, password, identifiers): +def extract_encrypted(base_dir, password, identifiers, preserve_timestamp): backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) print("Decrypting WhatsApp database...", end="") try: @@ -70,7 +70,9 @@ def is_encrypted(base_dir): return False -def extract_media(base_dir, identifiers): +def extract_media(base_dir, identifiers, preserve_timestamp): + if preserve_timestamp: + from Whatsapp_Chat_Exporter.bplist import BPListReader if is_encrypted(base_dir): if not support_encrypted: print("You don't have the dependencies to handle encrypted backup.") @@ -110,6 +112,7 @@ def extract_media(base_dir, identifiers): c.execute(f"""SELECT fileID, relativePath, flags, + file AS metadata, ROW_NUMBER() OVER(ORDER BY relativePath) AS _index FROM Files WHERE domain = '{_wts_id}' @@ -132,6 +135,11 @@ def extract_media(base_dir, identifiers): pass elif flags == 1: shutil.copyfile(os.path.join(base_dir, folder, hashes), destination) + if preserve_timestamp: + metadata = BPListReader(row["metadata"]).parse() + creation = metadata["$objects"][1]["Birth"] + modification = metadata["$objects"][1]["LastModified"] + os.utime(destination, (modification, modification)) if row["_index"] % 100 == 0: print(f"Extracting WhatsApp files...({row['_index']}/{total_row_number})", end="\r") row = c.fetchone() From 380289d1c1f80d7ad81591d678cc9fefe16da31f Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Oct 2023 10:12:58 +0800 Subject: [PATCH 15/24] Bug fix on missing argument #65 --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index 2fc6e5c..af7b1bd 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -81,7 +81,7 @@ def extract_media(base_dir, identifiers, preserve_timestamp): return False print("Encryption detected on the backup!") password = getpass.getpass("Enter the password for the backup:") - extract_encrypted(base_dir, password, identifiers) + extract_encrypted(base_dir, password, identifiers, preserve_timestamp) else: wts_db = os.path.join(base_dir, identifiers.MESSAGE[:2], identifiers.MESSAGE) contact_db = os.path.join(base_dir, identifiers.CONTACT[:2], identifiers.CONTACT) From 8eb05424fdf05ddf3aa681863976161f4f24faaf Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Oct 2023 11:42:39 +0800 Subject: [PATCH 16/24] Support preservation of file timestamp for encrypted backup --- Whatsapp_Chat_Exporter/extract_iphone_media.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract_iphone_media.py b/Whatsapp_Chat_Exporter/extract_iphone_media.py index af7b1bd..8fa0f65 100644 --- a/Whatsapp_Chat_Exporter/extract_iphone_media.py +++ b/Whatsapp_Chat_Exporter/extract_iphone_media.py @@ -16,7 +16,7 @@ else: support_encrypted = True -def extract_encrypted(base_dir, password, identifiers, preserve_timestamp): +def extract_encrypted(base_dir, password, identifiers, bplist_reader=None): backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False) print("Decrypting WhatsApp database...", end="") try: @@ -37,7 +37,7 @@ def extract_encrypted(base_dir, password, identifiers, preserve_timestamp): print("Done") extract_thread = threading.Thread( target=backup.extract_files_by_domain, - args=(identifiers.DOMAIN, identifiers.DOMAIN) + args=(identifiers.DOMAIN, identifiers.DOMAIN, bplist_reader) ) extract_thread.daemon = True extract_thread.start() @@ -70,9 +70,10 @@ def is_encrypted(base_dir): return False -def extract_media(base_dir, identifiers, preserve_timestamp): +def extract_media(base_dir, identifiers, preserve_timestamp=False): if preserve_timestamp: - from Whatsapp_Chat_Exporter.bplist import BPListReader + from Whatsapp_Chat_Exporter.bplist import BPListReader + preserve_timestamp = BPListReader if is_encrypted(base_dir): if not support_encrypted: print("You don't have the dependencies to handle encrypted backup.") From be4adadbd8f0ce051708747b08cd421246f2b433 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 8 Oct 2023 11:42:47 +0800 Subject: [PATCH 17/24] Update __main__.py --- Whatsapp_Chat_Exporter/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 01354bf..13985a8 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -21,7 +21,7 @@ except ImportError: def main(): parser = ArgumentParser( description = 'A customizable Android and iPhone WhatsApp database parser that ' - 'will give you the history of your WhatsApp conversations inHTML ' + 'will give you the history of your WhatsApp conversations in HTML ' 'and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.', epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT' ) From fc50415afdb8f21e79a2d30cc481dfaebac969db Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 24 Oct 2023 00:05:15 +0800 Subject: [PATCH 18/24] An attempt to fix #67 --- Whatsapp_Chat_Exporter/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 2b5fb90..2fb0e3a 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -316,7 +316,7 @@ def messages(db, data, media_folder): i += 1 content = c.fetchone() continue - if "-" in content["key_remote_jid"] and content["key_from_me"] == 0: + if (content["chat_subject"] is not None or "-" in content["key_remote_jid"]) and content["key_from_me"] == 0: name = fallback = None if table_message: if content["sender_jid_row_id"] > 0: From 3082c83bc44c41c715ff215ba5d988a9ab9d514f Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Tue, 24 Oct 2023 00:16:49 +0800 Subject: [PATCH 19/24] Convert all links to hyperlinks #69 --- Whatsapp_Chat_Exporter/whatsapp.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Whatsapp_Chat_Exporter/whatsapp.html b/Whatsapp_Chat_Exporter/whatsapp.html index 6731aa1..72c9b6b 100644 --- a/Whatsapp_Chat_Exporter/whatsapp.html +++ b/Whatsapp_Chat_Exporter/whatsapp.html @@ -139,7 +139,7 @@ {% else %} {% if msg.media == false %} - {{ msg.data | sanitize_except() }} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} {% else %} {% if "image/" in msg.mime %} @@ -162,7 +162,7 @@ {% endif %} {% if msg.caption is not none %}
- {{ msg.caption }} + {{ msg.caption | urlize(none, true, '_blank') }} {% endif %} {% endif %} {% endif %} @@ -223,7 +223,7 @@ {% else %} {% if msg.media == false %} - {{ msg.data | sanitize_except() }} + {{ msg.data | sanitize_except() | urlize(none, true, '_blank') }} {% else %} {% if "image/" in msg.mime %} @@ -246,7 +246,7 @@ {% endif %} {% if msg.caption is not none %}
- {{ msg.caption }} + {{ msg.caption | urlize(none, true, '_blank') }} {% endif %} {% endif %} {% endif %} From efea1d6165cd0cfe79ded965808e17127bc2d792 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:52:58 +0800 Subject: [PATCH 20/24] Remove waste --- Whatsapp_Chat_Exporter/extract.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 2fb0e3a..37a6b61 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -175,7 +175,6 @@ def messages(db, data, media_folder): total_row_number = c.fetchone()[0] print(f"Processing messages...(0/{total_row_number})", end="\r") - phone_number_re = re.compile(r"[0-9]+@s.whatsapp.net") try: c.execute("""SELECT messages.key_remote_jid, messages._id, From efd5ed80b2c987a3c1a1076d3ec0a73af631ecfa Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:56:19 +0800 Subject: [PATCH 21/24] Bug fix on ignoring system chat --- Whatsapp_Chat_Exporter/extract.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 37a6b61..9a569d5 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -458,6 +458,7 @@ def media(db, data, media_folder): ON message_media.message_row_id = messages._id LEFT JOIN media_hash_thumbnail ON message_media.file_hash = media_hash_thumbnail.media_hash + WHERE jid.type <> 7 ORDER BY messages.key_remote_jid ASC""" ) except sqlite3.OperationalError: @@ -478,6 +479,7 @@ def media(db, data, media_folder): ON jid._id = chat.jid_row_id LEFT JOIN media_hash_thumbnail ON message_media.file_hash = media_hash_thumbnail.media_hash + WHERE jid.type <> 7 ORDER BY jid.raw_string ASC""" ) content = c.fetchone() From 7da71e84fefaaff3ec53fab5a7bcfd50fe52d2d4 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 29 Oct 2023 18:03:40 +0800 Subject: [PATCH 22/24] Bug fix on missing sender in group chats #67 --- Whatsapp_Chat_Exporter/extract.py | 10 ++++++---- Whatsapp_Chat_Exporter/utility.py | 10 +++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 9a569d5..99aee80 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -15,7 +15,7 @@ from base64 import b64decode, b64encode from Whatsapp_Chat_Exporter.data_model import ChatStore, Message from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, determine_metadata, get_status_location from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template -from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS +from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, JidType try: import zlib @@ -197,7 +197,8 @@ def messages(db, data, media_folder): message_system.action_type, message_system_group.is_me_joined, jid_old.raw_string as old_jid, - jid_new.raw_string as new_jid + jid_new.raw_string as new_jid, + jid_global.type as jid_type, FROM messages LEFT JOIN messages_quotes ON messages.quoted_row_id = messages_quotes._id @@ -243,7 +244,8 @@ def messages(db, data, media_folder): message_system.action_type, message_system_group.is_me_joined, jid_old.raw_string as old_jid, - jid_new.raw_string as new_jid + jid_new.raw_string as new_jid, + jid_global.type as jid_type, FROM message LEFT JOIN message_quoted ON message_quoted.message_row_id = message._id @@ -315,7 +317,7 @@ def messages(db, data, media_folder): i += 1 content = c.fetchone() continue - if (content["chat_subject"] is not None or "-" in content["key_remote_jid"]) and content["key_from_me"] == 0: + if content["jid_type"] == JidType.GROUP and content["key_from_me"] == 0: name = fallback = None if table_message: if content["sender_jid_row_id"] > 0: diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 26f1df1..6490081 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -7,13 +7,15 @@ from datetime import datetime from enum import IntEnum from Whatsapp_Chat_Exporter.data_model import ChatStore try: - from enum import StrEnum + from enum import StrEnum, IntEnum except ImportError: # < Python 3.11 from enum import Enum class StrEnum(str, Enum): pass + class IntEnum(int, Enum): + pass MAX_SIZE = 4 * 1024 * 1024 # Default 4MB ROW_SIZE = 0x3D0 @@ -302,3 +304,9 @@ class WhatsAppBusinessIdentifier(StrEnum): MESSAGE = "724bd3b98b18518b455a87c1f3ac3a0d189c4466" CONTACT = "d7246a707f51ddf8b17ee2dddabd9e0a4da5c552" DOMAIN = "AppDomainGroup-group.net.whatsapp.WhatsAppSMB.shared" + +class JidType(IntEnum): + PM = 0 + GROUP = 1 + SYSTEM_BROADCAST = 5 + STATUS = 11 From 86ea938323f3c0992444afdabeddb0290dc7208a Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 29 Oct 2023 18:05:02 +0800 Subject: [PATCH 23/24] Add fields for implementing read receipt --- Whatsapp_Chat_Exporter/extract.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 99aee80..54cc74e 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -199,6 +199,11 @@ def messages(db, data, media_folder): jid_old.raw_string as old_jid, jid_new.raw_string as new_jid, jid_global.type as jid_type, + group_concat(receipt_user.receipt_timestamp) as receipt_timestamp, + group_concat(message.received_timestamp) as received_timestamp, + group_concat(receipt_user.read_timestamp) as read_timestamp, + group_concat(receipt_user.played_timestamp) as played_timestamp, + group_concat(messages.read_device_timestamp) as read_device_timestamp FROM messages LEFT JOIN messages_quotes ON messages.quoted_row_id = messages_quotes._id @@ -218,7 +223,10 @@ def messages(db, data, media_folder): ON jid_old._id = message_system_number_change.old_jid_row_id LEFT JOIN jid jid_new ON jid_new._id = message_system_number_change.new_jid_row_id - WHERE messages.key_remote_jid <> '-1';""" + LEFT JOIN receipt_user + ON receipt_user.message_row_id = messages._id + WHERE messages.key_remote_jid <> '-1' + GROUP BY message._id;""" ) except sqlite3.OperationalError: try: @@ -246,6 +254,10 @@ def messages(db, data, media_folder): jid_old.raw_string as old_jid, jid_new.raw_string as new_jid, jid_global.type as jid_type, + group_concat(receipt_user.receipt_timestamp) as receipt_timestamp, + group_concat(message.received_timestamp) as received_timestamp, + group_concat(receipt_user.read_timestamp) as read_timestamp, + group_concat(receipt_user.played_timestamp) as played_timestamp FROM message LEFT JOIN message_quoted ON message_quoted.message_row_id = message._id @@ -275,7 +287,10 @@ def messages(db, data, media_folder): ON jid_old._id = message_system_number_change.old_jid_row_id LEFT JOIN jid jid_new ON jid_new._id = message_system_number_change.new_jid_row_id - WHERE key_remote_jid <> '-1';""" + LEFT JOIN receipt_user + ON receipt_user.message_row_id = message._id + WHERE key_remote_jid <> '-1' + GROUP BY message._id;""" ) except Exception as e: raise e From 150180fdff5279e834e927861534aff7ffd52e8c Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 29 Oct 2023 18:05:45 +0800 Subject: [PATCH 24/24] Refactor --- Whatsapp_Chat_Exporter/extract.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Whatsapp_Chat_Exporter/extract.py b/Whatsapp_Chat_Exporter/extract.py index 54cc74e..e0ada4e 100644 --- a/Whatsapp_Chat_Exporter/extract.py +++ b/Whatsapp_Chat_Exporter/extract.py @@ -192,7 +192,7 @@ def messages(db, data, media_folder): messages.key_id, messages_quotes.data as quoted_data, messages.media_caption, - missed_call_logs.video_call, + missed_call_logs.video_call, chat.subject as chat_subject, message_system.action_type, message_system_group.is_me_joined, @@ -406,8 +406,7 @@ def messages(db, data, media_folder): message.data = None else: # Real message - if content["media_wa_type"] == 20: # Sticker is a message - message.sticker = True + message.sticker = content["media_wa_type"] == 20 # Sticker is a message if content["key_from_me"] == 1: if content["status"] == 5 and content["edit_version"] == 7 or table_message and content["media_wa_type"] == 15: msg = "Message deleted" @@ -488,13 +487,13 @@ def media(db, data, media_folder): file_hash, thumbnail FROM message_media - INNER JOIN message - ON message_media.message_row_id = message._id - LEFT JOIN chat - ON chat._id = message.chat_row_id - INNER JOIN jid - ON jid._id = chat.jid_row_id - LEFT JOIN media_hash_thumbnail + INNER JOIN message + ON message_media.message_row_id = message._id + LEFT JOIN chat + ON chat._id = message.chat_row_id + INNER JOIN jid + ON jid._id = chat.jid_row_id + LEFT JOIN media_hash_thumbnail ON message_media.file_hash = media_hash_thumbnail.media_hash WHERE jid.type <> 7 ORDER BY jid.raw_string ASC"""