Skip to content

Commit f29085d

Browse files
BeercowBeercow
committed
v2024.09.20
1 parent 8296858 commit f29085d

File tree

10 files changed

+153
-95
lines changed

10 files changed

+153
-95
lines changed

‎OneDriveExplorer/Images/splashv.png

659 Bytes
Loading

‎OneDriveExplorer/OneDriveExplorer.py

+9-9
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
)
5151

5252
__author__ = "Brian Maloney"
53-
__version__ = "2024.07.24"
53+
__version__ = "2024.09.20"
5454
__email__ = "bmmaloney97@gmail.com"
5555
rbin = []
5656
DATParser = dat_parser.DATParser()
@@ -205,10 +205,10 @@ def output():
205205
name = f'{sql_find[0][0]}_{sql_find[0][1]}'
206206
except Exception:
207207
name = 'SQLite_DB'
208-
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account = SQLiteParser.parse_sql(args.sql)
208+
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account, localHashAlgorithm = SQLiteParser.parse_sql(args.sql)
209209

210210
if not df.empty:
211-
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, args.sql, rbin_df, account, args.reghive, args.RECYCLE_BIN)
211+
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, args.sql, rbin_df, account, args.reghive, args.RECYCLE_BIN, localHashAlgorithm)
212212

213213
if df.empty:
214214
print(f'Unable to parse {name} sqlite database.')
@@ -231,10 +231,10 @@ def output():
231231
account = os.path.dirname(args.file.replace('/', '\\')).rsplit('\\', 1)[-1]
232232
name = os.path.split(args.file)[1]
233233

234-
df, rbin_df, df_scope, scopeID = DATParser.parse_dat(args.file, account)
234+
df, rbin_df, df_scope, scopeID, localHashAlgorithm = DATParser.parse_dat(args.file, account)
235235

236236
if not df.empty:
237-
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, args.file, rbin_df, account, args.reghive, args.RECYCLE_BIN)
237+
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, args.file, rbin_df, account, args.reghive, args.RECYCLE_BIN, localHashAlgorithm)
238238

239239
if df.empty:
240240
filename = args.file.replace('/', '\\')
@@ -313,10 +313,10 @@ def output():
313313
account = os.path.dirname(filename.replace('/', '\\')).rsplit('\\', 1)[-1]
314314
name = os.path.split(filename)[1]
315315

316-
df, rbin_df, df_scope, scopeID = DATParser.parse_dat(filename, account)
316+
df, rbin_df, df_scope, scopeID, localHashAlgorithm = DATParser.parse_dat(filename, account)
317317

318318
if not df.empty:
319-
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, filename, rbin_df, account, args.reghive, args.RECYCLE_BIN)
319+
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, filename, rbin_df, account, args.reghive, args.RECYCLE_BIN, localHashAlgorithm)
320320

321321
if df.empty:
322322
filename = filename.replace('/', '\\')
@@ -330,10 +330,10 @@ def output():
330330
for account, sql_dir in v.items():
331331
name = f'{key}_{account}'
332332

333-
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account = SQLiteParser.parse_sql(sql_dir)
333+
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account, localHashAlgorithm = SQLiteParser.parse_sql(sql_dir)
334334

335335
if not df.empty:
336-
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, sql_dir, rbin_df, account, args.reghive, args.RECYCLE_BIN)
336+
cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, sql_dir, rbin_df, account, args.reghive, args.RECYCLE_BIN, localHashAlgorithm)
337337

338338
if df.empty:
339339
print(f'Unable to parse {name} sqlite database.')

‎OneDriveExplorer/OneDriveExplorer_GUI.py

+54-27
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
import keyboard
5151
from ruamel.yaml import YAML
5252
import logging
53-
from io import StringIO as StringBuffer
5453
from datetime import datetime
5554
from cerberus import Validator
5655
import warnings
@@ -95,15 +94,15 @@
9594
WS_MINIMIZEBOX = 131072
9695
WS_MAXIMIZEBOX = 65536
9796

98-
log_capture_string = StringBuffer()
97+
log_capture_string = StringIO()
9998
logging.basicConfig(level=logging.INFO,
10099
format='%(asctime)s, %(levelname)s, %(message)s',
101100
datefmt='%Y-%m-%d %H:%M:%S',
102101
handlers=[logging.StreamHandler(log_capture_string)]
103102
)
104103

105104
__author__ = "Brian Maloney"
106-
__version__ = "2024.07.24"
105+
__version__ = "2024.09.20"
107106
__email__ = "bmmaloney97@gmail.com"
108107
rbin = []
109108
user_logs = {}
@@ -1291,10 +1290,11 @@ def highlight_pattern(self, pattern, tag, start="1.0", end="end", regexp=False):
12911290

12921291
class Result:
12931292

1294-
def __init__(self, master, *args, folder=True, tags=''):
1293+
def __init__(self, master, *args, folder=True, folderShared='', tags=''):
12951294
self.master = master
12961295
self.args = args
12971296
self.folder = folder
1297+
self.folderShared = folderShared
12981298
self.tags = tags
12991299
self.type = []
13001300
self.status = []
@@ -1360,7 +1360,7 @@ def handle_folder_status(self, num, values_list):
13601360
''
13611361
)
13621362

1363-
if num == '7' and len(values_list) > 11:
1363+
if num == '7' and len(values_list) > 12:
13641364
shortcut_item = next((item for item in self.args[0] if 'shortcutitemindex:' in item.lower()), None)
13651365
if shortcut_item and int(shortcut_item.split(' ')[1]) > 0:
13661366
self.type.clear()
@@ -1372,6 +1372,14 @@ def handle_folder_status(self, num, values_list):
13721372
else:
13731373
self.type.append(self.get_type_image(num))
13741374

1375+
sharedItem = next(
1376+
(item.split(' ')[1] for item in self.args[0] if 'shareditem:' in item.lower() and len(item.split(' ')) > 1),
1377+
''
1378+
)
1379+
1380+
if sharedItem == '1':
1381+
self.status.append(shared_big_img)
1382+
13751383
if not set(self.lock_list).intersection(spoPermissions):
13761384
if num not in ('10', '11'):
13771385
self.status.append(locked_big_img)
@@ -1397,7 +1405,7 @@ def process_non_folder_status(self, values_list):
13971405
else:
13981406
self.status.append(self.get_status_image(num))
13991407

1400-
if sharedItem == '1':
1408+
if sharedItem == '1' or self.folderShared == '1':
14011409
self.status.append(shared_big_img)
14021410

14031411
if not set(self.lock_list).intersection(spoPermissions) and not any('inrecyclebin:' in item.lower() for item in self.args[0]):
@@ -2086,7 +2094,7 @@ def get_info(self, event): # need to look into click performance
20862094
# find logs for files/folders
20872095
if any('status:' in value.lower() for value in values):
20882096
# Find the item containing 'resourceID:' and extract the desired part
2089-
resourceID = next((value.split(" ")[1].split("+")[0] for value in values if 'resourceid:' in value.lower()), '')
2097+
resourceID = next((value.split(" ")[1].split("+")[0] for value in values if value.lower().startswith('resourceid:')), '')
20902098
# Concatenate DataFrames containing the resource_id
20912099
info = pd.concat([df.loc[df.Params.astype('string').str.contains(f'{resourceID}', case=False, na=False)] for df in df_list])
20922100

@@ -2182,6 +2190,14 @@ def file_pane(self):
21822190
else:
21832191
self.status.append(image_mapping.get(folderStatus, online_img))
21842192

2193+
sharedItemF = next(
2194+
(item.split(' ')[1] for item in values if 'shareditem:' in item.lower() and len(item.split(' ')) > 1),
2195+
''
2196+
)
2197+
2198+
if sharedItemF == '1':
2199+
self.status.append(shared_img)
2200+
21852201
if not set(lock_list).intersection(spoPermissions) and str(tags) != 'red':
21862202
if folderStatus not in ('10', '11', ''):
21872203
self.status.append(locked_img)
@@ -2193,6 +2209,11 @@ def file_pane(self):
21932209

21942210
try:
21952211
if cur_item[0] in file_items:
2212+
folderShared = next(
2213+
(item.split(' ')[1] for item in self.tv.item(cur_item[0])["values"] if 'shareditem:' in item.lower() and len(item.split(' ')) > 1),
2214+
''
2215+
)
2216+
21962217
for i in file_items[cur_item[0]]:
21972218
self.status.clear()
21982219
item_data_i = self.tv.item(i)
@@ -2229,7 +2250,7 @@ def file_pane(self):
22292250

22302251
self.status.append(image_mapping.get(fileStatus, online_img))
22312252

2232-
if sharedItem == '1':
2253+
if sharedItem == '1' or folderShared == '1':
22332254
self.status.append(shared_img)
22342255

22352256
if not set(lock_list).intersection(spoPermissions_i) and str(tags_i) != 'red':
@@ -3059,14 +3080,18 @@ def search(item=''):
30593080
image_key = tv.item(child, 'image')[0]
30603081
Result(root, values, child, image_key)
30613082
if child in file_items:
3083+
folderShared = next(
3084+
(item.split(' ')[1] for item in tv.item(child, 'values') if 'shareditem:' in item.lower() and len(item.split(' ')) > 1),
3085+
''
3086+
)
30623087
for i in file_items[child]:
30633088
if query.lower() in str(tv.item(i, 'values')).lower():
30643089
tags = ''
30653090
if tv.item(i, 'tags'):
30663091
tags = 'red'
30673092
values = tv.item(i, 'values')
30683093
image_key = tv.item(i, 'image')[0]
3069-
Result(root, values, i, image_key, folder=False, tags=tags)
3094+
Result(root, values, i, image_key, folder=False, folderShared=folderShared, tags=tags)
30703095
search(item=child)
30713096

30723097

@@ -3532,16 +3557,11 @@ def odl(folder_name, csv=False):
35323557
file_manager.tv2.delete(*file_manager.tv2.get_children())
35333558
file_manager.tv3.delete(*file_manager.tv3.get_children())
35343559
key_find = re.compile(r'Users/(?P<user>.*)?/AppData')
3535-
if csv:
3536-
key = folder_name.name.split('/')[-1].split('_')[0]
3537-
else:
3538-
key = re.findall(key_find, folder_name)
3539-
if len(key) == 0:
3540-
key = 'ODL'
3541-
else:
3542-
key = key[0]
35433560
pb.stop()
3561+
start = time.time()
3562+
35443563
if csv:
3564+
key = folder_name.name.split('/')[-1].split('_')[0]
35453565
header_list = ['Filename',
35463566
'File_Index',
35473567
'Timestamp',
@@ -3576,6 +3596,11 @@ def odl(folder_name, csv=False):
35763596
odl = pd.DataFrame()
35773597
logging.error(f'{folder_name.name} not a valid ODL csv.')
35783598
else:
3599+
key = re.findall(key_find, folder_name)
3600+
if len(key) == 0:
3601+
key = 'ODL'
3602+
else:
3603+
key = key[0]
35793604
odl = parse_odl(folder_name, key, pb, value_label, gui=True)
35803605

35813606
tb = ttk.Frame()
@@ -3606,7 +3631,7 @@ def odl(folder_name, csv=False):
36063631

36073632
pb.stop()
36083633
pb.configure(mode='determinate')
3609-
value_label['text'] = "Parsing complete"
3634+
value_label['text'] = f'Parsing complete. {format((time.time() - start), ".4f")} seconds'
36103635

36113636
mcount = (len(log_capture_string.getvalue().split('\n')) - 1)
36123637
message['text'] = mcount
@@ -3656,7 +3681,7 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
36563681
account = os.path.dirname(filename.replace('/', '\\')).rsplit('\\', 1)[-1]
36573682
name = os.path.split(filename)[1]
36583683

3659-
df, rbin_df, df_scope, scopeID = DATParser.parse_dat(filename, account,
3684+
df, rbin_df, df_scope, scopeID, localHashAlgorithm = DATParser.parse_dat(filename, account,
36603685
gui=True, pb=pb,
36613686
value_label=value_label)
36623687

@@ -3668,13 +3693,14 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
36683693
rbin_df, account,
36693694
reghive,
36703695
recbin,
3696+
localHashAlgorithm=localHashAlgorithm,
36713697
gui=True,
36723698
pb=pb,
36733699
value_label=value_label)
36743700

36753701
dat = True
36763702

3677-
if x == 'Load from SQLite':
3703+
elif x == 'Load from SQLite':
36783704
filename = filename.replace('/', '\\')
36793705
sql_dir = re.compile(r'\\Users\\(?P<user>.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?P<account>.*?)$')
36803706
sql_find = re.findall(sql_dir, filename)
@@ -3686,7 +3712,7 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
36863712
pb.configure(mode='indeterminate')
36873713
value_label['text'] = 'Building folder list. Please wait....'
36883714
pb.start()
3689-
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account = SQLiteParser.parse_sql(filename)
3715+
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account, localHashAlgorithm = SQLiteParser.parse_sql(filename)
36903716

36913717
if not df.empty:
36923718
cache, rbin_df = OneDriveParser.parse_onedrive(df,
@@ -3698,18 +3724,19 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
36983724
account,
36993725
reghive,
37003726
recbin,
3727+
localHashAlgorithm=localHashAlgorithm,
37013728
gui=True,
37023729
pb=pb,
37033730
value_label=value_label)
37043731
pb.stop()
37053732
dat = True
37063733

3707-
if x == 'Import JSON':
3734+
elif x == 'Import JSON':
37083735
cache = json.load(filename)
37093736
df = pd.DataFrame()
37103737
rbin_df = pd.DataFrame()
37113738

3712-
if x == 'Import CSV':
3739+
elif x == 'Import CSV':
37133740
account = ''
37143741
df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID = parse_csv(filename)
37153742

@@ -3722,11 +3749,12 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
37223749
account,
37233750
reghive,
37243751
recbin,
3752+
localHashAlgorithm=False,
37253753
gui=True,
37263754
pb=pb,
37273755
value_label=value_label)
37283756

3729-
if x == 'Project':
3757+
elif x == 'Project':
37303758
name = filename
37313759
pass
37323760

@@ -3741,11 +3769,10 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, live=False):
37413769
if x == 'Import JSON':
37423770
parent_child(cache, None, True)
37433771
df_GraphMetadata_Records = pd.DataFrame(dfs_to_concat)
3744-
else:
3745-
parent_child(cache)
3746-
if x == 'Import JSON':
37473772
curItem = tv.get_children()[-1]
37483773
file_count, del_count, folder_count = json_count(item=curItem)
3774+
else:
3775+
parent_child(cache)
37493776

37503777
pb.stop()
37513778
pb.configure(mode='determinate')

‎OneDriveExplorer/ode/parsers/csv_file.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
def parse_csv(filename):
3333

3434
file = open(filename.name, 'r', encoding='utf-8')
35-
columns_to_drop = ['parentResourceId', 'resourceId', 'inRecycleBin', 'volumeId', 'fileId', 'DeleteTimeStamp', 'notificationTime', 'hash']
36-
columns_to_drop_2 = ['MountPoint', 'Path', 'fileName', 'graphMetadataJSON', 'spoCompositeID', 'createdBy', 'modifiedBy', 'filePolicies', 'fileExtension', 'lastWriteCount']
35+
columns_to_drop = ['parentResourceId', 'resourceId', 'inRecycleBin', 'volumeId', 'fileId', 'DeleteTimeStamp', 'notificationTime', 'hash', 'deletingProcess']
36+
columns_to_drop_2 = ['MountPoint', 'fileName', 'graphMetadataJSON', 'spoCompositeID', 'createdBy', 'modifiedBy', 'filePolicies', 'fileExtension', 'lastWriteCount']
3737

3838
dtypes = {'Type': 'object',
3939
'scopeID': 'object',
@@ -93,7 +93,7 @@ def parse_csv(filename):
9393
'deletingProcess']]
9494
rbin_df = rbin_df.astype(object)
9595
rbin_df = rbin_df.where(pd.notna(rbin_df), '')
96-
df = df.drop(df[df['Type'] == 'Deleteed'].index)
96+
df = df.drop(df[df['Type'] == 'Deleted'].index)
9797
df.drop(columns=columns_to_drop, inplace=True)
9898
else:
9999
rbin_df = pd.DataFrame()
@@ -110,7 +110,6 @@ def parse_csv(filename):
110110
df = df.astype(object)
111111
df = df.where(pd.notna(df), None)
112112
df.drop(columns=columns_to_drop_2, inplace=True)
113-
114113
except Exception as e:
115114
print(e)
116115
log.error(f'Not a valid csv. {csv_name}')

‎OneDriveExplorer/ode/parsers/dat.py

+7-2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ def __init__(self):
4343
self.log = logging.getLogger(__name__)
4444
self.datstruct = cstruct.cstruct()
4545
self.DAT_DEF = f'{self.application_path}/ode/helpers/structures'
46+
self.localHashAlgorithm = 0
4647
self.datstruct.loadfile(self.DAT_DEF)
4748
self.dict_1 = {'lastChange': 0,
4849
'sharedItem': 0,
@@ -371,6 +372,8 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
371372
block = self.datstruct.DAT_FOLDER_v29_v2c(f.read(chunk))
372373
else:
373374
block = self.datstruct.DAT_FOLDER_v2d_v36(f.read(chunk))
375+
block._values.update([('sharedItem', '')])
376+
print(dict(block._values))
374377

375378
elif ff == '09':
376379
data_type = 'Scope'
@@ -506,12 +509,14 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
506509
df_files = df_files.drop(columns=columns)
507510
if account == 'Personal':
508511
df_files['localHashDigest'] = df_files['localHashDigest'].apply(lambda x: f'SHA1({x})')
512+
self.localHashAlgorithm = 4
509513
else:
510514
df_files['localHashDigest'] = df_files['localHashDigest'].apply(lambda x: f'quickXor({codecs.encode(binascii.unhexlify(x), "base64").decode("utf-8").rstrip()})')
515+
self.localHashAlgorithm = 5
511516
df_files['size'] = df_files['size'].apply(lambda x: '0 KB' if x == 0 else f'{x//1024 + 1:,} KB')
512517
df_files['spoPermissions'] = df_files['spoPermissions'].apply(lambda x: permissions(x))
513518
df_files['lastChange'] = pd.to_datetime(df_files['lastChange'], unit='s').astype(str)
514-
df_folders = pd.read_csv(temp_folders, usecols=['parentScopeID', 'parentResourceID', 'resourceID', 'eTag', 'folderName', 'folderStatus', 'spoPermissions', 'volumeID', 'itemIndex'])
519+
df_folders = pd.read_csv(temp_folders, usecols=['parentScopeID', 'parentResourceID', 'resourceID', 'eTag', 'folderName', 'folderStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'sharedItem'])
515520
temp_folders.close()
516521
df_folders.insert(0, 'Type', 'Folder')
517522
df_folders.rename(columns={"folderName": "Name"}, inplace=True)
@@ -521,4 +526,4 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
521526
df = pd.concat([df_scope, df_files, df_folders], ignore_index=True, axis=0)
522527
df = df.where(pd.notnull(df), None)
523528

524-
return df, pd.DataFrame(), df_scope, scopeID
529+
return df, pd.DataFrame(), df_scope, scopeID, self.localHashAlgorithm

0 commit comments

Comments
 (0)