mirror of
https://github.com/KnugiHK/WhatsApp-Chat-Exporter.git
synced 2026-04-23 14:31:37 +00:00
Comply with PEP8 with some exception
E501 and F401 are not fully enforced
This commit is contained in:
KnugiHK
143
extract.py
143
extract.py
@@ -1,11 +1,9 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
import json
|
||||
import jinja2
|
||||
import os
|
||||
import base64
|
||||
import requests
|
||||
import shutil
|
||||
import re
|
||||
@@ -21,6 +19,7 @@ def determine_day(last, current):
|
||||
else:
|
||||
return current
|
||||
|
||||
|
||||
def contacts(db, data):
|
||||
# Get contacts
|
||||
c = db.cursor()
|
||||
@@ -31,9 +30,10 @@ def contacts(db, data):
|
||||
c.execute("""SELECT jid, display_name FROM wa_contacts; """)
|
||||
row = c.fetchone()
|
||||
while row is not None:
|
||||
data[row[0]] = {"name": row[1], "messages":{}}
|
||||
data[row[0]] = {"name": row[1], "messages": {}}
|
||||
row = c.fetchone()
|
||||
|
||||
|
||||
def messages(db, data):
|
||||
# Get message history
|
||||
c = db.cursor()
|
||||
@@ -42,7 +42,25 @@ def messages(db, data):
|
||||
print(f"Gathering messages...(0/{total_row_number})", end="\r")
|
||||
|
||||
phone_number_re = re.compile(r"[0-9]+@s.whatsapp.net")
|
||||
c.execute("""SELECT messages.key_remote_jid, messages._id, messages.key_from_me, messages.timestamp, messages.data, messages.status, messages.edit_version, messages.thumb_image, messages.remote_resource, messages.media_wa_type, messages.latitude, messages.longitude, messages_quotes.key_id as quoted, messages.key_id, messages_quotes.data, messages.media_caption FROM messages LEFT JOIN messages_quotes ON messages.quoted_row_id = messages_quotes._id; """)
|
||||
c.execute("""SELECT messages.key_remote_jid,
|
||||
messages._id,
|
||||
messages.key_from_me,
|
||||
messages.timestamp,
|
||||
messages.data,
|
||||
messages.status,
|
||||
messages.edit_version,
|
||||
messages.thumb_image,
|
||||
messages.remote_resource,
|
||||
messages.media_wa_type,
|
||||
messages.latitude,
|
||||
messages.longitude,
|
||||
messages_quotes.key_id as quoted,
|
||||
messages.key_id,
|
||||
messages_quotes.data,
|
||||
messages.media_caption
|
||||
FROM messages
|
||||
LEFT JOIN messages_quotes
|
||||
ON messages.quoted_row_id = messages_quotes._id;""")
|
||||
i = 0
|
||||
content = c.fetchone()
|
||||
while content is not None:
|
||||
@@ -65,30 +83,33 @@ def messages(db, data):
|
||||
fallback = None
|
||||
else:
|
||||
fallback = None
|
||||
|
||||
data[content[0]]["messages"][content[1]]["sender"] = name or fallback
|
||||
|
||||
data[content[0]]["messages"][content[1]
|
||||
]["sender"] = name or fallback
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["sender"] = None
|
||||
|
||||
|
||||
if content[12] is not None:
|
||||
data[content[0]]["messages"][content[1]]["reply"] = content[12]
|
||||
data[content[0]]["messages"][content[1]]["quoted_data"] = content[14]
|
||||
data[content[0]]["messages"][content[1]
|
||||
]["quoted_data"] = content[14]
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["reply"] = None
|
||||
|
||||
|
||||
if content[15] is not None:
|
||||
data[content[0]]["messages"][content[1]]["caption"] = content[15]
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["caption"] = None
|
||||
|
||||
|
||||
if content[5] == 6:
|
||||
if "-" in content[0]:
|
||||
# Is Group
|
||||
if content[4] is not None:
|
||||
try:
|
||||
int(content[4])
|
||||
except:
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{The group name changed to "f"{content[4]}"" }"
|
||||
except ValueError:
|
||||
msg = "{The group name changed to "f"{content[4]}"" }"
|
||||
data[content[0]]["messages"][content[1]]["data"] = msg
|
||||
else:
|
||||
del data[content[0]]["messages"][content[1]]
|
||||
else:
|
||||
@@ -96,7 +117,8 @@ def messages(db, data):
|
||||
if thumb_image is not None:
|
||||
if b"\x00\x00\x01\x74\x00\x1A" in thumb_image:
|
||||
# Add user
|
||||
added = phone_number_re.search(thumb_image.decode("unicode_escape"))[0]
|
||||
added = phone_number_re.search(
|
||||
thumb_image.decode("unicode_escape"))[0]
|
||||
if added in data:
|
||||
name_right = data[added]["name"]
|
||||
else:
|
||||
@@ -106,14 +128,15 @@ def messages(db, data):
|
||||
name_left = data[content[8]]["name"]
|
||||
else:
|
||||
name_left = content[8].split('@')[0]
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{"f"{name_left}"f" added {name_right or 'You'}""}"
|
||||
msg = "{"f"{name_left}"f" added {name_right or 'You'}""}"
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{"f"Added {name_right or 'You'}""}"
|
||||
if b"\xac\xed\x00\x05\x74\x00" in thumb_image:
|
||||
msg = "{"f"Added {name_right or 'You'}""}"
|
||||
elif b"\xac\xed\x00\x05\x74\x00" in thumb_image:
|
||||
# Changed number
|
||||
original = content[8].split('@')[0]
|
||||
changed = thumb_image[7:].decode().split('@')[0]
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{"f"{original} changed to {changed}""}"
|
||||
msg = "{"f"{original} changed to {changed}""}"
|
||||
data[content[0]]["messages"][content[1]]["data"] = msg
|
||||
else:
|
||||
if content[4] is None:
|
||||
del data[content[0]]["messages"][content[1]]
|
||||
@@ -121,30 +144,34 @@ def messages(db, data):
|
||||
# Private chat
|
||||
if content[4] is None and content[7] is None:
|
||||
del data[content[0]]["messages"][content[1]]
|
||||
|
||||
|
||||
else:
|
||||
if content[2] == 1:
|
||||
if content[5] == 5 and content[6] == 7:
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{Message deleted}"
|
||||
msg = "{Message deleted}"
|
||||
else:
|
||||
if content[9] == "5":
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{ Location shared: "f"{content[10], content[11]}"" }"
|
||||
msg = "{ Location shared: "f"{content[10], content[11]}"" }"
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["data"] = content[4]
|
||||
msg = content[4]
|
||||
else:
|
||||
if content[5] == 0 and content[6] == 7:
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{Message deleted}"
|
||||
msg = "{Message deleted}"
|
||||
else:
|
||||
if content[9] == "5":
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{ Location shared: "f"{content[10], content[11]}"" }"
|
||||
msg = "{ Location shared: "f"{content[10], content[11]}"" }"
|
||||
else:
|
||||
data[content[0]]["messages"][content[1]]["data"] = content[4]
|
||||
|
||||
msg = content[4]
|
||||
|
||||
data[content[0]]["messages"][content[1]]["data"] = msg
|
||||
|
||||
i += 1
|
||||
if i % 1000 == 0:
|
||||
print(f"Gathering messages...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(
|
||||
f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
|
||||
def media(db, data, media_folder):
|
||||
# Get media
|
||||
@@ -153,7 +180,16 @@ def media(db, data, media_folder):
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"\nGathering media...(0/{total_row_number})", end="\r")
|
||||
i = 0
|
||||
c.execute("""SELECT messages.key_remote_jid, message_row_id, file_path, message_url, mime_type, media_key FROM message_media INNER JOIN messages ON message_media.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
c.execute("""SELECT messages.key_remote_jid,
|
||||
message_row_id,
|
||||
file_path,
|
||||
message_url,
|
||||
mime_type,
|
||||
media_key
|
||||
FROM message_media
|
||||
INNER JOIN messages
|
||||
ON message_media.message_row_id = messages._id
|
||||
ORDER BY messages.key_remote_jid ASC""")
|
||||
content = c.fetchone()
|
||||
mime = MimeTypes()
|
||||
while content is not None:
|
||||
@@ -171,26 +207,36 @@ def media(db, data, media_folder):
|
||||
data[content[0]]["messages"][content[1]]["mime"] = content[4]
|
||||
else:
|
||||
# if "https://mmg" in content[4]:
|
||||
# try:
|
||||
# r = requests.get(content[3])
|
||||
# if r.status_code != 200:
|
||||
# raise RuntimeError()
|
||||
# except:
|
||||
# data[content[0]]["messages"][content[1]]["data"] = "{The media is missing}"
|
||||
# data[content[0]]["messages"][content[1]]["media"] = True
|
||||
# data[content[0]]["messages"][content[1]]["mime"] = "media"
|
||||
# try:
|
||||
# r = requests.get(content[3])
|
||||
# if r.status_code != 200:
|
||||
# raise RuntimeError()
|
||||
# except:
|
||||
# data[content[0]]["messages"][content[1]]["data"] = "{The media is missing}"
|
||||
# data[content[0]]["messages"][content[1]]["media"] = True
|
||||
# data[content[0]]["messages"][content[1]]["mime"] = "media"
|
||||
# else:
|
||||
data[content[0]]["messages"][content[1]]["data"] = "{The media is missing}"
|
||||
data[content[0]]["messages"][content[1]
|
||||
]["data"] = "{The media is missing}"
|
||||
data[content[0]]["messages"][content[1]]["mime"] = "media"
|
||||
i += 1
|
||||
if i % 100 == 0:
|
||||
print(f"Gathering media...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(
|
||||
f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
|
||||
def vcard(db, data):
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT message_row_id, messages.key_remote_jid, vcard, messages.media_name FROM messages_vcards INNER JOIN messages ON messages_vcards.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
c.execute("""SELECT message_row_id,
|
||||
messages.key_remote_jid,
|
||||
vcard,
|
||||
messages.media_name
|
||||
FROM messages_vcards
|
||||
INNER JOIN messages
|
||||
ON messages_vcards.message_row_id = messages._id
|
||||
ORDER BY messages.key_remote_jid ASC;""")
|
||||
rows = c.fetchall()
|
||||
total_row_number = len(rows)
|
||||
print(f"\nGathering vCards...(0/{total_row_number})", end="\r")
|
||||
@@ -203,10 +249,13 @@ def vcard(db, data):
|
||||
if not os.path.isfile(file_path):
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(row[2])
|
||||
data[row[1]]["messages"][row[0]]["data"] = row[3] + "{ The vCard file cannot be displayed here, however it should be located at " + file_path + "}"
|
||||
data[row[1]]["messages"][row[0]]["data"] = row[3] + \
|
||||
"{ The vCard file cannot be displayed here, however it " \
|
||||
"should be located at " + file_path + "}"
|
||||
data[row[1]]["messages"][row[0]]["mime"] = "x-vcard"
|
||||
print(f"Gathering vCards...({index + 1}/{total_row_number})", end="\r")
|
||||
|
||||
|
||||
def create_html(data, output_folder):
|
||||
templateLoader = jinja2.FileSystemLoader(searchpath="./")
|
||||
templateEnv = jinja2.Environment(loader=templateLoader)
|
||||
@@ -224,11 +273,11 @@ def create_html(data, output_folder):
|
||||
if len(data[i]["messages"]) == 0:
|
||||
continue
|
||||
phone_number = i.split('@')[0]
|
||||
if "-"in i:
|
||||
if "-" in i:
|
||||
file_name = ""
|
||||
else:
|
||||
file_name = phone_number
|
||||
|
||||
|
||||
if data[i]["name"] is not None:
|
||||
if file_name != "":
|
||||
file_name += "-"
|
||||
@@ -237,12 +286,14 @@ def create_html(data, output_folder):
|
||||
else:
|
||||
name = phone_number
|
||||
safe_file_name = ''
|
||||
safe_file_name = "".join(x for x in file_name if x.isalnum() or x in "- ")
|
||||
safe_file_name = "".join(
|
||||
x for x in file_name if x.isalnum() or x in "- ")
|
||||
with open(f"{output_folder}/{safe_file_name}.html", "w", encoding="utf-8") as f:
|
||||
f.write(template.render(name=name, msgs=data[i]["messages"].values(), my_avatar=None, their_avatar=f"WhatsApp/Avatars/{i}.j"))
|
||||
f.write(template.render(name=name, msgs=data[i]["messages"].values(
|
||||
), my_avatar=None, their_avatar=f"WhatsApp/Avatars/{i}.j"))
|
||||
if current % 10 == 0:
|
||||
print(f"Creating HTML...({current}/{total_row_number})", end="\r")
|
||||
|
||||
|
||||
print(f"Creating HTML...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
|
||||
@@ -267,7 +318,7 @@ if __name__ == "__main__":
|
||||
# "--template",
|
||||
# dest="html",
|
||||
# default="wa.db",
|
||||
# help="Path to HTML template")
|
||||
# help="Path to HTML template")
|
||||
(options, args) = parser.parse_args()
|
||||
msg_db = "msgstore.db"
|
||||
output_folder = "temp"
|
||||
@@ -279,7 +330,7 @@ if __name__ == "__main__":
|
||||
elif len(args) == 2:
|
||||
msg_db = args[0]
|
||||
output_folder = args[1]
|
||||
|
||||
|
||||
data = {}
|
||||
|
||||
if os.path.isfile(contact_db):
|
||||
|
||||
Reference in New Issue
Block a user