mirror of
https://github.com/KnugiHK/WhatsApp-Chat-Exporter.git
synced 2026-04-26 16:01:33 +00:00
Modular scripts
This commit is contained in:
KnugiHK
153
extract.py
153
extract.py
@@ -21,12 +21,9 @@ def determine_day(last, current):
|
||||
else:
|
||||
return current
|
||||
|
||||
data = {}
|
||||
|
||||
# Get contacts
|
||||
if os.path.isfile("wa.db"):
|
||||
wa = sqlite3.connect("wa.db")
|
||||
c = wa.cursor()
|
||||
def contacts(db, data):
|
||||
# Get contacts
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT count() FROM wa_contacts""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering contacts...({total_row_number})")
|
||||
@@ -36,20 +33,19 @@ if os.path.isfile("wa.db"):
|
||||
while row is not None:
|
||||
data[row[0]] = {"name": row[1], "messages":{}}
|
||||
row = c.fetchone()
|
||||
wa.close()
|
||||
|
||||
# Get message history
|
||||
msg = sqlite3.connect("msgstore.db")
|
||||
c = msg.cursor()
|
||||
c.execute("""SELECT count() FROM messages""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering messages...(0/{total_row_number})", end="\r")
|
||||
def messages(db, data):
|
||||
# Get message history
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT count() FROM messages""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering messages...(0/{total_row_number})", end="\r")
|
||||
|
||||
phone_number_re = re.compile(r"[0-9]+@s.whatsapp.net")
|
||||
c.execute("""SELECT messages.key_remote_jid, messages._id, messages.key_from_me, messages.timestamp, messages.data, messages.status, messages.edit_version, messages.thumb_image, messages.remote_resource, messages.media_wa_type, messages.latitude, messages.longitude, messages_quotes.key_id as quoted, messages.key_id, messages_quotes.data, messages.media_caption FROM messages LEFT JOIN messages_quotes ON messages.quoted_row_id = messages_quotes._id; """)
|
||||
i = 0
|
||||
content = c.fetchone()
|
||||
while content is not None:
|
||||
phone_number_re = re.compile(r"[0-9]+@s.whatsapp.net")
|
||||
c.execute("""SELECT messages.key_remote_jid, messages._id, messages.key_from_me, messages.timestamp, messages.data, messages.status, messages.edit_version, messages.thumb_image, messages.remote_resource, messages.media_wa_type, messages.latitude, messages.longitude, messages_quotes.key_id as quoted, messages.key_id, messages_quotes.data, messages.media_caption FROM messages LEFT JOIN messages_quotes ON messages.quoted_row_id = messages_quotes._id; """)
|
||||
i = 0
|
||||
content = c.fetchone()
|
||||
while content is not None:
|
||||
if content[0] not in data:
|
||||
data[content[0]] = {"name": None, "messages": {}}
|
||||
data[content[0]]["messages"][content[1]] = {
|
||||
@@ -148,18 +144,20 @@ while content is not None:
|
||||
if i % 1000 == 0:
|
||||
print(f"Gathering messages...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
# Get media
|
||||
print(f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
c.execute("""SELECT count() FROM message_media""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"\nGathering media...(0/{total_row_number})", end="\r")
|
||||
i = 0
|
||||
c.execute("""SELECT messages.key_remote_jid, message_row_id, file_path, message_url, mime_type, media_key FROM message_media INNER JOIN messages ON message_media.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
content = c.fetchone()
|
||||
mime = MimeTypes()
|
||||
while content is not None:
|
||||
file_path = f"WhatsApp/{content[2]}"
|
||||
def media(db, data, media_folder):
|
||||
# Get media
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT count() FROM message_media""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"\nGathering media...(0/{total_row_number})", end="\r")
|
||||
i = 0
|
||||
c.execute("""SELECT messages.key_remote_jid, message_row_id, file_path, message_url, mime_type, media_key FROM message_media INNER JOIN messages ON message_media.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
content = c.fetchone()
|
||||
mime = MimeTypes()
|
||||
while content is not None:
|
||||
file_path = f"{media_folder}/{content[2]}"
|
||||
data[content[0]]["messages"][content[1]]["media"] = True
|
||||
if os.path.isfile(file_path):
|
||||
data[content[0]]["messages"][content[1]]["data"] = file_path
|
||||
@@ -188,14 +186,16 @@ while content is not None:
|
||||
if i % 100 == 0:
|
||||
print(f"Gathering media...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
c.execute("""SELECT message_row_id, messages.key_remote_jid, vcard, messages.media_name FROM messages_vcards INNER JOIN messages ON messages_vcards.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
rows = c.fetchall()
|
||||
total_row_number = len(rows)
|
||||
print(f"\nGathering vCards...(0/{total_row_number})", end="\r")
|
||||
base = "WhatsApp/vCards"
|
||||
for index, row in enumerate(rows):
|
||||
def vcard(db, data):
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT message_row_id, messages.key_remote_jid, vcard, messages.media_name FROM messages_vcards INNER JOIN messages ON messages_vcards.message_row_id = messages._id ORDER BY messages.key_remote_jid ASC""")
|
||||
rows = c.fetchall()
|
||||
total_row_number = len(rows)
|
||||
print(f"\nGathering vCards...(0/{total_row_number})", end="\r")
|
||||
base = "WhatsApp/vCards"
|
||||
for index, row in enumerate(rows):
|
||||
if not os.path.isdir(base):
|
||||
os.mkdir(base)
|
||||
file_name = "".join(x for x in row[3] if x.isalnum())
|
||||
@@ -207,24 +207,20 @@ for index, row in enumerate(rows):
|
||||
data[row[1]]["messages"][row[0]]["mime"] = "x-vcard"
|
||||
print(f"Gathering vCards...({index + 1}/{total_row_number})", end="\r")
|
||||
|
||||
templateLoader = jinja2.FileSystemLoader(searchpath="./")
|
||||
templateEnv = jinja2.Environment(loader=templateLoader)
|
||||
templateEnv.globals.update(determine_day=determine_day)
|
||||
TEMPLATE_FILE = "whatsapp.html"
|
||||
template = templateEnv.get_template(TEMPLATE_FILE)
|
||||
def create_html(data, output_folder):
|
||||
templateLoader = jinja2.FileSystemLoader(searchpath="./")
|
||||
templateEnv = jinja2.Environment(loader=templateLoader)
|
||||
templateEnv.globals.update(determine_day=determine_day)
|
||||
TEMPLATE_FILE = "whatsapp.html"
|
||||
template = templateEnv.get_template(TEMPLATE_FILE)
|
||||
|
||||
total_row_number = len(data)
|
||||
print(f"\nCreating HTML...(0/{total_row_number})", end="\r")
|
||||
total_row_number = len(data)
|
||||
print(f"\nCreating HTML...(0/{total_row_number})", end="\r")
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
output_folder = "temp"
|
||||
else:
|
||||
output_folder = sys.argv[2]
|
||||
|
||||
if not os.path.isdir(output_folder):
|
||||
if not os.path.isdir(output_folder):
|
||||
os.mkdir(output_folder)
|
||||
|
||||
for current, i in enumerate(data):
|
||||
for current, i in enumerate(data):
|
||||
if len(data[i]["messages"]) == 0:
|
||||
continue
|
||||
phone_number = i.split('@')[0]
|
||||
@@ -247,14 +243,61 @@ for current, i in enumerate(data):
|
||||
if current % 10 == 0:
|
||||
print(f"Creating HTML...({current}/{total_row_number})", end="\r")
|
||||
|
||||
print(f"Creating HTML...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(f"Creating HTML...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
if not os.path.isdir(f"{output_folder}/WhatsApp"):
|
||||
shutil.move("WhatsApp", f"{output_folder}/")
|
||||
|
||||
with open("result.json", "w") as f:
|
||||
if __name__ == "__main__":
|
||||
from optparse import OptionParser
|
||||
parser = OptionParser()
|
||||
parser.add_option(
|
||||
"-w",
|
||||
"--wa",
|
||||
dest="wa",
|
||||
default="wa.db",
|
||||
help="Path to contact database")
|
||||
parser.add_option(
|
||||
"-m",
|
||||
"--media",
|
||||
dest="media",
|
||||
default="WhatsApp",
|
||||
help="Path to WhatsApp media folder"
|
||||
)
|
||||
# parser.add_option(
|
||||
# "-t",
|
||||
# "--template",
|
||||
# dest="html",
|
||||
# default="wa.db",
|
||||
# help="Path to HTML template")
|
||||
(options, args) = parser.parse_args()
|
||||
msg_db = "msgstore.db"
|
||||
output_folder = "temp"
|
||||
contact_db = options.wa
|
||||
media_folder = options.media
|
||||
|
||||
if len(args) == 1:
|
||||
msg_db = args[0]
|
||||
elif len(args) == 2:
|
||||
msg_db = args[0]
|
||||
output_folder = args[1]
|
||||
|
||||
data = {}
|
||||
|
||||
if os.path.isfile(contact_db):
|
||||
with sqlite3.connect(contact_db) as db:
|
||||
contacts(db, data)
|
||||
if os.path.isfile(msg_db):
|
||||
with sqlite3.connect(msg_db) as db:
|
||||
messages(db, data)
|
||||
media(db, data, media_folder)
|
||||
vcard(db, data)
|
||||
create_html(data, output_folder)
|
||||
|
||||
if not os.path.isdir(f"{output_folder}/WhatsApp"):
|
||||
shutil.move(media_folder, f"{output_folder}/")
|
||||
|
||||
with open("result.json", "w") as f:
|
||||
data = json.dumps(data)
|
||||
print(f"\nWriting JSON file...({int(len(data)/1024/1024)}MB)")
|
||||
f.write(data)
|
||||
|
||||
print("Everything is done!")
|
||||
print("Everything is done!")
|
||||
|
||||
@@ -11,6 +11,7 @@ import shutil
|
||||
from datetime import datetime
|
||||
from mimetypes import MimeTypes
|
||||
|
||||
APPLE_TIME = datetime.timestamp(datetime(2001,1,1))
|
||||
|
||||
def determine_day(last, current):
|
||||
last = datetime.fromtimestamp(last).date()
|
||||
@@ -20,34 +21,31 @@ def determine_day(last, current):
|
||||
else:
|
||||
return current
|
||||
|
||||
data = {}
|
||||
def messages(db, data):
|
||||
c = db.cursor()
|
||||
# Get contacts
|
||||
c.execute("""SELECT count() FROM ZWACHATSESSION""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering contacts...({total_row_number})")
|
||||
|
||||
# Get contacts
|
||||
msg = sqlite3.connect("7c7fba66680ef796b916b067077cc246adacf01d")
|
||||
c = msg.cursor()
|
||||
c.execute("""SELECT count() FROM ZWACHATSESSION""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering contacts...({total_row_number})")
|
||||
|
||||
c.execute("""SELECT ZCONTACTJID, ZPARTNERNAME FROM ZWACHATSESSION; """)
|
||||
row = c.fetchone()
|
||||
while row is not None:
|
||||
c.execute("""SELECT ZCONTACTJID, ZPARTNERNAME FROM ZWACHATSESSION; """)
|
||||
row = c.fetchone()
|
||||
while row is not None:
|
||||
data[row[0]] = {"name": row[1], "messages":{}}
|
||||
row = c.fetchone()
|
||||
|
||||
# Get message history
|
||||
c.execute("""SELECT count() FROM ZWAMESSAGE""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
apple_time = datetime.timestamp(datetime(2001,1,1))
|
||||
print(f"Gathering messages...(0/{total_row_number})", end="\r")
|
||||
# Get message history
|
||||
c.execute("""SELECT count() FROM ZWAMESSAGE""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"Gathering messages...(0/{total_row_number})", end="\r")
|
||||
|
||||
c.execute("""SELECT COALESCE(ZFROMJID, ZTOJID), ZWAMESSAGE.Z_PK, ZISFROMME, ZMESSAGEDATE, ZTEXT, ZMESSAGETYPE, ZWAGROUPMEMBER.ZMEMBERJID FROM main.ZWAMESSAGE LEFT JOIN main.ZWAGROUPMEMBER ON main.ZWAMESSAGE.ZGROUPMEMBER = main.ZWAGROUPMEMBER.Z_PK;""")
|
||||
i = 0
|
||||
content = c.fetchone()
|
||||
while content is not None:
|
||||
c.execute("""SELECT COALESCE(ZFROMJID, ZTOJID), ZWAMESSAGE.Z_PK, ZISFROMME, ZMESSAGEDATE, ZTEXT, ZMESSAGETYPE, ZWAGROUPMEMBER.ZMEMBERJID FROM main.ZWAMESSAGE LEFT JOIN main.ZWAGROUPMEMBER ON main.ZWAMESSAGE.ZGROUPMEMBER = main.ZWAGROUPMEMBER.Z_PK;""")
|
||||
i = 0
|
||||
content = c.fetchone()
|
||||
while content is not None:
|
||||
if content[0] not in data:
|
||||
data[content[0]] = {"name": None, "messages": {}}
|
||||
ts = apple_time + content[3]
|
||||
ts = APPLE_TIME + content[3]
|
||||
data[content[0]]["messages"][content[1]] = {
|
||||
"from_me": bool(content[2]),
|
||||
"timestamp": ts,
|
||||
@@ -103,17 +101,19 @@ while content is not None:
|
||||
if i % 1000 == 0:
|
||||
print(f"Gathering messages...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
# Get media
|
||||
print(f"Gathering messages...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
c.execute("""SELECT count() FROM ZWAMEDIAITEM""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"\nGathering media...(0/{total_row_number})", end="\r")
|
||||
i = 0
|
||||
c.execute("""SELECT COALESCE(ZWAMESSAGE.ZFROMJID, ZWAMESSAGE.ZTOJID) as _id, ZMESSAGE, ZMEDIALOCALPATH, ZMEDIAURL, ZVCARDSTRING, ZMEDIAKEY, ZTITLE FROM ZWAMEDIAITEM INNER JOIN ZWAMESSAGE ON ZWAMEDIAITEM.ZMESSAGE = ZWAMESSAGE.Z_PK WHERE ZMEDIALOCALPATH IS NOT NULL ORDER BY _id ASC""")
|
||||
content = c.fetchone()
|
||||
mime = MimeTypes()
|
||||
while content is not None:
|
||||
def media(db, data, media_folder):
|
||||
c = db.cursor()
|
||||
# Get media
|
||||
c.execute("""SELECT count() FROM ZWAMEDIAITEM""")
|
||||
total_row_number = c.fetchone()[0]
|
||||
print(f"\nGathering media...(0/{total_row_number})", end="\r")
|
||||
i = 0
|
||||
c.execute("""SELECT COALESCE(ZWAMESSAGE.ZFROMJID, ZWAMESSAGE.ZTOJID) as _id, ZMESSAGE, ZMEDIALOCALPATH, ZMEDIAURL, ZVCARDSTRING, ZMEDIAKEY, ZTITLE FROM ZWAMEDIAITEM INNER JOIN ZWAMESSAGE ON ZWAMEDIAITEM.ZMESSAGE = ZWAMESSAGE.Z_PK WHERE ZMEDIALOCALPATH IS NOT NULL ORDER BY _id ASC""")
|
||||
content = c.fetchone()
|
||||
mime = MimeTypes()
|
||||
while content is not None:
|
||||
file_path = f"Message/{content[2]}"
|
||||
data[content[0]]["messages"][content[1]]["media"] = True
|
||||
|
||||
@@ -145,14 +145,16 @@ while content is not None:
|
||||
if i % 100 == 0:
|
||||
print(f"Gathering media...({i}/{total_row_number})", end="\r")
|
||||
content = c.fetchone()
|
||||
print(f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(f"Gathering media...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
c.execute("""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, ZWAMEDIAITEM.ZMESSAGE, COALESCE(ZWAMESSAGE.ZFROMJID, ZWAMESSAGE.ZTOJID) as _id, ZVCARDNAME, ZVCARDSTRING FROM ZWAVCARDMENTION INNER JOIN ZWAMEDIAITEM ON ZWAVCARDMENTION.ZMEDIAITEM = ZWAMEDIAITEM.Z_PK INNER JOIN ZWAMESSAGE ON ZWAMEDIAITEM.ZMESSAGE = ZWAMESSAGE.Z_PK""")
|
||||
rows = c.fetchall()
|
||||
total_row_number = len(rows)
|
||||
print(f"\nGathering vCards...(0/{total_row_number})", end="\r")
|
||||
base = "Message/vCards"
|
||||
for index, row in enumerate(rows):
|
||||
def vcard(db, data):
|
||||
c = db.cursor()
|
||||
c.execute("""SELECT DISTINCT ZWAVCARDMENTION.ZMEDIAITEM, ZWAMEDIAITEM.ZMESSAGE, COALESCE(ZWAMESSAGE.ZFROMJID, ZWAMESSAGE.ZTOJID) as _id, ZVCARDNAME, ZVCARDSTRING FROM ZWAVCARDMENTION INNER JOIN ZWAMEDIAITEM ON ZWAVCARDMENTION.ZMEDIAITEM = ZWAMEDIAITEM.Z_PK INNER JOIN ZWAMESSAGE ON ZWAMEDIAITEM.ZMESSAGE = ZWAMESSAGE.Z_PK""")
|
||||
rows = c.fetchall()
|
||||
total_row_number = len(rows)
|
||||
print(f"\nGathering vCards...(0/{total_row_number})", end="\r")
|
||||
base = "Message/vCards"
|
||||
for index, row in enumerate(rows):
|
||||
if not os.path.isdir(base):
|
||||
os.mkdir(base)
|
||||
file_name = "".join(x for x in row[3] if x.isalnum())
|
||||
@@ -165,24 +167,20 @@ for index, row in enumerate(rows):
|
||||
data[row[2]]["messages"][row[1]]["media"] = True
|
||||
print(f"Gathering vCards...({index + 1}/{total_row_number})", end="\r")
|
||||
|
||||
templateLoader = jinja2.FileSystemLoader(searchpath="./")
|
||||
templateEnv = jinja2.Environment(loader=templateLoader)
|
||||
templateEnv.globals.update(determine_day=determine_day)
|
||||
TEMPLATE_FILE = "whatsapp.html"
|
||||
template = templateEnv.get_template(TEMPLATE_FILE)
|
||||
def create_html(data, output_folder):
|
||||
templateLoader = jinja2.FileSystemLoader(searchpath="./")
|
||||
templateEnv = jinja2.Environment(loader=templateLoader)
|
||||
templateEnv.globals.update(determine_day=determine_day)
|
||||
TEMPLATE_FILE = "whatsapp.html"
|
||||
template = templateEnv.get_template(TEMPLATE_FILE)
|
||||
|
||||
total_row_number = len(data)
|
||||
print(f"\nCreating HTML...(0/{total_row_number})", end="\r")
|
||||
total_row_number = len(data)
|
||||
print(f"\nCreating HTML...(0/{total_row_number})", end="\r")
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
output_folder = "temp"
|
||||
else:
|
||||
output_folder = sys.argv[2]
|
||||
|
||||
if not os.path.isdir(output_folder):
|
||||
if not os.path.isdir(output_folder):
|
||||
os.mkdir(output_folder)
|
||||
|
||||
for current, i in enumerate(data):
|
||||
for current, i in enumerate(data):
|
||||
if len(data[i]["messages"]) == 0:
|
||||
continue
|
||||
phone_number = i.split('@')[0]
|
||||
@@ -206,14 +204,57 @@ for current, i in enumerate(data):
|
||||
if current % 10 == 0:
|
||||
print(f"Creating HTML...({current}/{total_row_number})", end="\r")
|
||||
|
||||
print(f"Creating HTML...({total_row_number}/{total_row_number})", end="\r")
|
||||
print(f"Creating HTML...({total_row_number}/{total_row_number})", end="\r")
|
||||
|
||||
if not os.path.isdir(f"{output_folder}/Message"):
|
||||
shutil.move("Message", f"{output_folder}/")
|
||||
if __name__ == "__main__":
|
||||
from optparse import OptionParser
|
||||
parser = OptionParser()
|
||||
parser.add_option(
|
||||
"-w",
|
||||
"--wa",
|
||||
dest="wa",
|
||||
default="wa.db",
|
||||
help="Path to contact database")
|
||||
parser.add_option(
|
||||
"-m",
|
||||
"--media",
|
||||
dest="media",
|
||||
default="Message",
|
||||
help="Path to WhatsApp media folder"
|
||||
)
|
||||
# parser.add_option(
|
||||
# "-t",
|
||||
# "--template",
|
||||
# dest="html",
|
||||
# default="wa.db",
|
||||
# help="Path to HTML template")
|
||||
(options, args) = parser.parse_args()
|
||||
msg_db = "7c7fba66680ef796b916b067077cc246adacf01d"
|
||||
output_folder = "temp"
|
||||
contact_db = options.wa
|
||||
media_folder = options.media
|
||||
|
||||
with open("result.json", "w") as f:
|
||||
if len(args) == 1:
|
||||
msg_db = args[0]
|
||||
elif len(args) == 2:
|
||||
msg_db = args[0]
|
||||
output_folder = args[1]
|
||||
|
||||
data = {}
|
||||
|
||||
if os.path.isfile(msg_db):
|
||||
with sqlite3.connect(msg_db) as db:
|
||||
messages(db, data)
|
||||
media(db, data, media_folder)
|
||||
vcard(db, data)
|
||||
create_html(data, output_folder)
|
||||
|
||||
if not os.path.isdir(f"{output_folder}/WhatsApp"):
|
||||
shutil.move(media_folder, f"{output_folder}/")
|
||||
|
||||
with open("result.json", "w") as f:
|
||||
data = json.dumps(data)
|
||||
print(f"\nWriting JSON file...({int(len(data)/1024/1024)}MB)")
|
||||
f.write(data)
|
||||
|
||||
print("Everything is done!")
|
||||
print("Everything is done!")
|
||||
|
||||
Reference in New Issue
Block a user