calvin.erfmann/WhatsApp-Chat-Exporter
1
0
Fork 0
mirror of https://github.com/KnugiHK/WhatsApp-Chat-Exporter.git synced 2026-04-25 23:41:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement empty chat filtering from SQL #112

Browse Source 
This commit also removed the old empty chat filtering logic.
This commit is contained in:
KnugiHK
2025-01-04 18:18:34 +08:00
parent 92d710bce8
commit 23af55d645
4 changed files with 39 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Whatsapp_Chat_Exporter/__main__.py
View File

@@ -17,8 +17,8 @@ else:
from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import exported_handler, android_handler
from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler
from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.data_model import ChatStore
from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, chat_is_empty, readable_to_bytes from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, readable_to_bytes, check_update
from Whatsapp_Chat_Exporter.utility import check_update, import_from_json, sanitize_filename, bytes_to_readable from Whatsapp_Chat_Exporter.utility import import_from_json, sanitize_filename, bytes_to_readable
from argparse import ArgumentParser, SUPPRESS from argparse import ArgumentParser, SUPPRESS
from datetime import datetime from datetime import datetime
from sys import exit from sys import exit
@@ -254,7 +254,9 @@ def main():
dest="filter_empty", dest="filter_empty",
default=True, default=True,
action='store_false', action='store_false',
help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." help=("By default, the exporter will not render chats with no valid message. "
"Setting this flag will cause the exporter to render those. "
"This is useful if chat(s) are missing from the output")
) )
parser.add_argument( parser.add_argument(
"--per-chat", "--per-chat",
@@ -504,9 +506,9 @@ def main():
if os.path.isfile(msg_db): if os.path.isfile(msg_db):
with sqlite3.connect(msg_db) as db: with sqlite3.connect(msg_db) as db:
db.row_factory = sqlite3.Row db.row_factory = sqlite3.Row
messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat) messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat, args.filter_empty)
media(db, data, args.media, args.filter_date, filter_chat, args.separate_media) media(db, data, args.media, args.filter_date, filter_chat, args.filter_empty, args.separate_media)
vcard(db, data, args.media, args.filter_date, filter_chat) vcard(db, data, args.media, args.filter_date, filter_chat, args.filter_empty)
if args.android: if args.android:
android_handler.calls(db, data, args.timezone_offset, filter_chat) android_handler.calls(db, data, args.timezone_offset, filter_chat)
elif args.ios and args.call_db_ios is not None: elif args.ios and args.call_db_ios is not None:
@@ -525,7 +527,6 @@ def main():
args.offline, args.offline,
args.size, args.size,
args.no_avatar, args.no_avatar,
args.filter_empty,
args.whatsapp_theme args.whatsapp_theme
) )
else: else:
@@ -563,7 +564,6 @@ def main():
args.offline, args.offline,
args.size, args.size,
args.no_avatar, args.no_avatar,
args.filter_empty,
args.whatsapp_theme args.whatsapp_theme
) )
for file in glob.glob(r'*.*'): for file in glob.glob(r'*.*'):
@@ -578,7 +578,6 @@ def main():
args.offline, args.offline,
args.size, args.size,
args.no_avatar, args.no_avatar,
args.filter_empty,
args.whatsapp_theme args.whatsapp_theme
) )
@@ -587,9 +586,6 @@ def main():
android_handler.create_txt(data, args.text_format) android_handler.create_txt(data, args.text_format)
if args.json and not args.import_json: if args.json and not args.import_json:
if args.filter_empty:
data = {k: v for k, v in data.items() if not chat_is_empty(v)}
if args.enrich_from_vcards is not None and not contact_store.is_empty(): if args.enrich_from_vcards is not None and not contact_store.is_empty():
contact_store.enrich_from_vcards(data) contact_store.enrich_from_vcards(data)

33
Whatsapp_Chat_Exporter/android_handler.py
View File

@@ -12,10 +12,10 @@ from hashlib import sha256
from base64 import b64decode, b64encode from base64 import b64decode, b64encode
from datetime import datetime from datetime import datetime
from Whatsapp_Chat_Exporter.data_model import ChatStore, Message from Whatsapp_Chat_Exporter.data_model import ChatStore, Message
from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata from Whatsapp_Chat_Exporter.utility import CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, DbType, convert_time_unit, determine_metadata, get_cond_for_empty
from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template, JidType from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template
from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location
from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, chat_is_empty from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify, bytes_to_readable, JidType
try: try:
import zlib import zlib
@@ -173,7 +173,7 @@ def contacts(db, data):
row = c.fetchone() row = c.fetchone()
def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty):
# Get message history # Get message history
c = db.cursor() c = db.cursor()
try: try:
@@ -181,7 +181,10 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
FROM messages FROM messages
INNER JOIN jid INNER JOIN jid
ON messages.key_remote_jid = jid.raw_string ON messages.key_remote_jid = jid.raw_string
LEFT JOIN chat
ON chat.jid_row_id = jid._id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")}
{f'AND timestamp {filter_date}' if filter_date is not None else ''} {f'AND timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")}""") {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid", "android")}""")
@@ -196,6 +199,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
LEFT JOIN jid jid_group LEFT JOIN jid jid_group
ON jid_group._id = message.sender_jid_row_id ON jid_group._id = message.sender_jid_row_id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")}
{f'AND timestamp {filter_date}' if filter_date is not None else ''} {f'AND timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""")
@@ -253,6 +257,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
LEFT JOIN receipt_user LEFT JOIN receipt_user
ON receipt_user.message_row_id = messages._id ON receipt_user.message_row_id = messages._id
WHERE messages.key_remote_jid <> '-1' WHERE messages.key_remote_jid <> '-1'
{get_cond_for_empty(filter_empty, "messages.key_remote_jid", "messages.needs_push")}
{f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")}
{get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global", "android")}
@@ -321,6 +326,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
LEFT JOIN receipt_user LEFT JOIN receipt_user
ON receipt_user.message_row_id = message._id ON receipt_user.message_row_id = message._id
WHERE key_remote_jid <> '-1' WHERE key_remote_jid <> '-1'
{get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")}
{f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")}
{get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")}
@@ -488,7 +494,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r") print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r")
def media(db, data, media_folder, filter_date, filter_chat, separate_media=True): def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=True):
# Get media # Get media
c = db.cursor() c = db.cursor()
try: try:
@@ -498,7 +504,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True)
ON message_media.message_row_id = messages._id ON message_media.message_row_id = messages._id
INNER JOIN jid INNER JOIN jid
ON messages.key_remote_jid = jid.raw_string ON messages.key_remote_jid = jid.raw_string
LEFT JOIN chat
ON chat.jid_row_id = jid._id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")}
{f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}""") {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}""")
@@ -514,6 +523,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True)
LEFT JOIN jid jid_group LEFT JOIN jid jid_group
ON jid_group._id = message.sender_jid_row_id ON jid_group._id = message.sender_jid_row_id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")}
{f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""") {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")}""")
@@ -536,7 +546,10 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True)
ON message_media.file_hash = media_hash_thumbnail.media_hash ON message_media.file_hash = media_hash_thumbnail.media_hash
INNER JOIN jid INNER JOIN jid
ON messages.key_remote_jid = jid.raw_string ON messages.key_remote_jid = jid.raw_string
LEFT JOIN chat
ON chat.jid_row_id = jid._id
WHERE jid.type <> 7 WHERE jid.type <> 7
{get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")}
{f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}
@@ -563,6 +576,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True)
LEFT JOIN jid jid_group LEFT JOIN jid jid_group
ON jid_group._id = message.sender_jid_row_id ON jid_group._id = message.sender_jid_row_id
WHERE jid.type <> 7 WHERE jid.type <> 7
{get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")}
{f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")}
@@ -613,7 +627,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True)
f"Processing media...({total_row_number}/{total_row_number})", end="\r") f"Processing media...({total_row_number}/{total_row_number})", end="\r")
def vcard(db, data, media_folder, filter_date, filter_chat): def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty):
c = db.cursor() c = db.cursor()
try: try:
c.execute(f"""SELECT message_row_id, c.execute(f"""SELECT message_row_id,
@@ -625,7 +639,10 @@ def vcard(db, data, media_folder, filter_date, filter_chat):
ON messages_vcards.message_row_id = messages._id ON messages_vcards.message_row_id = messages._id
INNER JOIN jid INNER JOIN jid
ON messages.key_remote_jid = jid.raw_string ON messages.key_remote_jid = jid.raw_string
LEFT JOIN chat
ON chat.jid_row_id = jid._id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")}
{f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")}
@@ -646,6 +663,7 @@ def vcard(db, data, media_folder, filter_date, filter_chat):
LEFT JOIN jid jid_group LEFT JOIN jid jid_group
ON jid_group._id = message.sender_jid_row_id ON jid_group._id = message.sender_jid_row_id
WHERE 1=1 WHERE 1=1
{get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")}
{f'AND message.timestamp {filter_date}' if filter_date is not None else ''} {f'AND message.timestamp {filter_date}' if filter_date is not None else ''}
{get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")}
{get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")} {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")}
@@ -760,7 +778,6 @@ def create_html(
offline_static=False, offline_static=False,
maximum_size=None, maximum_size=None,
no_avatar=False, no_avatar=False,
filter_empty=True,
experimental=False experimental=False
): ):
template = setup_template(template, no_avatar, experimental) template = setup_template(template, no_avatar, experimental)
@@ -775,8 +792,6 @@ def create_html(
for current, contact in enumerate(data): for current, contact in enumerate(data):
chat = data[contact] chat = data[contact]
if filter_empty and chat_is_empty(chat):
continue
safe_file_name, name = get_file_name(contact, chat) safe_file_name, name = get_file_name(contact, chat)
if maximum_size is not None: if maximum_size is not None:

6
Whatsapp_Chat_Exporter/ios_handler.py
View File

@@ -27,7 +27,7 @@ def contacts(db, data):
content = c.fetchone() content = c.fetchone()
def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat, filter_empty):
c = db.cursor() c = db.cursor()
# Get contacts # Get contacts
c.execute( c.execute(
@@ -227,7 +227,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
f"Processing messages...({total_row_number}/{total_row_number})", end="\r") f"Processing messages...({total_row_number}/{total_row_number})", end="\r")
def media(db, data, media_folder, filter_date, filter_chat, separate_media=False): def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=False):
c = db.cursor() c = db.cursor()
# Get media # Get media
c.execute(f"""SELECT count() c.execute(f"""SELECT count()
@@ -308,7 +308,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False
f"Processing media...({total_row_number}/{total_row_number})", end="\r") f"Processing media...({total_row_number}/{total_row_number})", end="\r")
def vcard(db, data, media_folder, filter_date, filter_chat): def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty):
c = db.cursor() c = db.cursor()
c.execute(f"""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, c.execute(f"""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM,
ZWAMEDIAITEM.ZMESSAGE, ZWAMEDIAITEM.ZMESSAGE,

10
Whatsapp_Chat_Exporter/utility.py
View File

@@ -220,6 +220,10 @@ def get_file_name(contact: str, chat: ChatStore):
return sanitize_filename(file_name), name return sanitize_filename(file_name), name
def get_cond_for_empty(enable, jid_field: str, broadcast_field: str):
return f"AND (chat.sort_timestamp IS NOT NULL OR {jid_field}='status@broadcast' OR {broadcast_field}>0)" if enable else ""
def get_chat_condition(filter, include, columns, jid=None, platform=None): def get_chat_condition(filter, include, columns, jid=None, platform=None):
if filter is not None: if filter is not None:
conditions = [] conditions = []
@@ -245,12 +249,6 @@ def get_chat_condition(filter, include, columns, jid=None, platform=None):
else: else:
return "" return ""
def _is_message_empty(message):
return (message.data is None or message.data == "") and not message.media
def chat_is_empty(chat: ChatStore):
return len(chat.messages) == 0 or all(_is_message_empty(message) for message in chat.messages.values())
# Android Specific # Android Specific
CRYPT14_OFFSETS = ( CRYPT14_OFFSETS = (