From fc9c76c34c3f28819fb24bae6168167bb7cb396d Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 15 Sep 2024 00:29:57 +0800 Subject: [PATCH] Implement chat filter with group msg sender for Android #85 --- Whatsapp_Chat_Exporter/android_handler.py | 62 ++++++++++++++--------- Whatsapp_Chat_Exporter/ios_handler.py | 28 +++++----- Whatsapp_Chat_Exporter/utility.py | 17 +++++-- 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 91f5d07..3ed763c 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -179,10 +179,12 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): try: c.execute(f"""SELECT count() FROM messages + INNER JOIN jid + ON messages.key_remote_jid = jid.raw_string WHERE 1=1 {f'AND timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "messages.key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "messages.key_remote_jid")}""") + {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid")} + {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid")}""") except sqlite3.OperationalError: c.execute(f"""SELECT count() @@ -191,10 +193,12 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): ON chat._id = message.chat_row_id INNER JOIN jid ON jid._id = chat.jid_row_id + LEFT JOIN jid jid_group + ON jid_group._id = message.sender_jid_row_id WHERE 1=1 {f'AND timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "jid.raw_string")} - {get_chat_condition(filter_chat[1], False, "jid.raw_string")}""") + {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid")}""") total_row_number = c.fetchone()[0] print(f"Processing messages...(0/{total_row_number})", end="\r") @@ -250,8 +254,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): ON receipt_user.message_row_id = messages._id WHERE messages.key_remote_jid <> '-1' {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "messages.key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "messages.key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global")} + {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "messages.remote_resource"], "jid_global")} GROUP BY messages._id ORDER BY messages.timestamp ASC;""" ) @@ -318,8 +322,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): ON receipt_user.message_row_id = message._id WHERE key_remote_jid <> '-1' {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global")} + {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global")} GROUP BY message._id;""" ) except Exception as e: @@ -492,10 +496,12 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) FROM message_media INNER JOIN messages ON message_media.message_row_id = messages._id + INNER JOIN jid + ON messages.key_remote_jid = jid.raw_string WHERE 1=1 {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "messages.key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "messages.key_remote_jid")}""") + {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid")} + {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid")}""") except sqlite3.OperationalError: c.execute(f"""SELECT count() FROM message_media @@ -505,10 +511,12 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON chat._id = message.chat_row_id INNER JOIN jid ON jid._id = chat.jid_row_id + LEFT JOIN jid jid_group + ON jid_group._id = message.sender_jid_row_id WHERE 1=1 {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "jid.raw_string")} - {get_chat_condition(filter_chat[1], False, "jid.raw_string")}""") + {get_chat_condition(filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid")}""") total_row_number = c.fetchone()[0] print(f"\nProcessing media...(0/{total_row_number})", end="\r") i = 0 @@ -530,8 +538,8 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON messages.key_remote_jid = jid.raw_string WHERE jid.type <> 7 {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "messages.key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "messages.key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid")} + {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid")} ORDER BY messages.key_remote_jid ASC""" ) except sqlite3.OperationalError: @@ -552,10 +560,12 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=True) ON jid._id = chat.jid_row_id LEFT JOIN media_hash_thumbnail ON message_media.file_hash = media_hash_thumbnail.media_hash + LEFT JOIN jid jid_group + ON jid_group._id = message.sender_jid_row_id WHERE jid.type <> 7 {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid")} ORDER BY jid.raw_string ASC""" ) content = c.fetchone() @@ -613,10 +623,12 @@ def vcard(db, data, media_folder, filter_date, filter_chat): FROM messages_vcards INNER JOIN messages ON messages_vcards.message_row_id = messages._id + INNER JOIN jid + ON messages.key_remote_jid = jid.raw_string WHERE 1=1 {f'AND messages.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "messages.key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "messages.key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid")} + {get_chat_condition(filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid")} ORDER BY messages.key_remote_jid ASC;""" ) except sqlite3.OperationalError: @@ -631,10 +643,12 @@ def vcard(db, data, media_folder, filter_date, filter_chat): ON chat._id = message.chat_row_id INNER JOIN jid ON jid._id = chat.jid_row_id + LEFT JOIN jid jid_group + ON jid_group._id = message.sender_jid_row_id WHERE 1=1 {f'AND message.timestamp {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "key_remote_jid")} - {get_chat_condition(filter_chat[1], False, "key_remote_jid")} + {get_chat_condition(filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid")} ORDER BY message.chat_row_id ASC;""" ) @@ -670,8 +684,8 @@ def calls(db, data, timezone_offset, filter_chat): LEFT JOIN chat ON call_log.jid_row_id = chat.jid_row_id WHERE 1=1 - {get_chat_condition(filter_chat[0], True, "jid.raw_string")} - {get_chat_condition(filter_chat[1], False, "jid.raw_string")}""") + {get_chat_condition(filter_chat[0], True, ["jid.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["jid.raw_string"], "jid")}""") total_row_number = c.fetchone()[0] if total_row_number == 0: return @@ -692,8 +706,8 @@ def calls(db, data, timezone_offset, filter_chat): LEFT JOIN chat ON call_log.jid_row_id = chat.jid_row_id WHERE 1=1 - {get_chat_condition(filter_chat[0], True, "jid.raw_string")} - {get_chat_condition(filter_chat[1], False, "jid.raw_string")}""" + {get_chat_condition(filter_chat[0], True, ["jid.raw_string"], "jid")} + {get_chat_condition(filter_chat[1], False, ["jid.raw_string"], "jid")}""" ) chat = ChatStore(Device.ANDROID, "WhatsApp Calls") content = c.fetchone() diff --git a/Whatsapp_Chat_Exporter/ios_handler.py b/Whatsapp_Chat_Exporter/ios_handler.py index 0eb970a..afd0784 100644 --- a/Whatsapp_Chat_Exporter/ios_handler.py +++ b/Whatsapp_Chat_Exporter/ios_handler.py @@ -32,8 +32,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): c.execute(f"""SELECT count() FROM ZWACHATSESSION WHERE 1=1 - {get_chat_condition(filter_chat[0], True, "ZWACHATSESSION.ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZWACHATSESSION.ZCONTACTJID")}""") + {get_chat_condition(filter_chat[0], True, ["ZWACHATSESSION.ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZWACHATSESSION.ZCONTACTJID"])}""") total_row_number = c.fetchone()[0] print(f"Processing contacts...({total_row_number})") @@ -45,8 +45,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): LEFT JOIN ZWAPROFILEPUSHNAME ON ZWACHATSESSION.ZCONTACTJID = ZWAPROFILEPUSHNAME.ZJID WHERE 1=1 - {get_chat_condition(filter_chat[0], True, "ZWACHATSESSION.ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZWACHATSESSION.ZCONTACTJID")};""" + {get_chat_condition(filter_chat[0], True, ["ZWACHATSESSION.ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZWACHATSESSION.ZCONTACTJID"])};""" ) content = c.fetchone() while content is not None: @@ -80,8 +80,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK WHERE 1=1 {f'AND ZMESSAGEDATE {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "ZWACHATSESSION.ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZWACHATSESSION.ZCONTACTJID")}""") + {get_chat_condition(filter_chat[0], True, ["ZWACHATSESSION.ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZWACHATSESSION.ZCONTACTJID"])}""") total_row_number = c.fetchone()[0] print(f"Processing messages...(0/{total_row_number})", end="\r") c.execute(f"""SELECT ZCONTACTJID, @@ -103,8 +103,8 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat): ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK WHERE 1=1 {f'AND ZMESSAGEDATE {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZCONTACTJID")} + {get_chat_condition(filter_chat[0], True, ["ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZCONTACTJID"])} ORDER BY ZMESSAGEDATE ASC;""") i = 0 content = c.fetchone() @@ -218,8 +218,8 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK WHERE 1=1 {f'AND ZMESSAGEDATE {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "ZWACHATSESSION.ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZWACHATSESSION.ZCONTACTJID")} + {get_chat_condition(filter_chat[0], True, ["ZWACHATSESSION.ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZWACHATSESSION.ZCONTACTJID"])} """) total_row_number = c.fetchone()[0] print(f"\nProcessing media...(0/{total_row_number})", end="\r") @@ -238,8 +238,8 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK WHERE ZMEDIALOCALPATH IS NOT NULL {f'AND ZWAMESSAGE.ZMESSAGEDATE {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZCONTACTJID")} + {get_chat_condition(filter_chat[0], True, ["ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZCONTACTJID"])} ORDER BY ZCONTACTJID ASC""") content = c.fetchone() mime = MimeTypes() @@ -311,8 +311,8 @@ def vcard(db, data, media_folder, filter_date, filter_chat): ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK WHERE 1=1 {f'AND ZWAMESSAGE.ZMESSAGEDATE {filter_date}' if filter_date is not None else ''} - {get_chat_condition(filter_chat[0], True, "ZCONTACTJID")} - {get_chat_condition(filter_chat[1], False, "ZCONTACTJID")};""") + {get_chat_condition(filter_chat[0], True, ["ZCONTACTJID"])} + {get_chat_condition(filter_chat[1], False, ["ZCONTACTJID"])};""") contents = c.fetchall() total_row_number = len(contents) print(f"\nProcessing vCards...(0/{total_row_number})", end="\r") diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index eda32bb..af636b3 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -199,12 +199,19 @@ def get_file_name(contact: str, chat: ChatStore): return sanitize_filename(file_name), name -def get_chat_condition(filter, include, column): +def get_chat_condition(filter, include, columns, jid=None): if filter is not None: - if include: - return f'''AND ({' OR '.join(f"{column} LIKE '%{chat}%'" for chat in filter)})''' - else: - return f'''AND ({' AND '.join(f"{column} NOT LIKE '%{chat}%'" for chat in filter)})''' + conditions = [] + for index, chat in enumerate(filter): + if include: + conditions.append(f"{' OR' if index > 0 else ''} {columns[0]} LIKE '%{chat}%'") + if len(columns) > 1: + conditions.append(f" OR ({columns[1]} LIKE '%{chat}%' AND {jid}.type == 1)") + else: + conditions.append(f"{' AND' if index > 0 else ''} {columns[0]} NOT LIKE '%{chat}%'") + if len(columns) > 1: + conditions.append(f" AND ({columns[1]} NOT LIKE '%{chat}%' AND {jid}.type == 1)") + return f"AND ({' '.join(conditions)})" else: return ""