-
Notifications
You must be signed in to change notification settings - Fork 0
/
upload.py
executable file
·178 lines (145 loc) · 6.04 KB
/
upload.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
#!/usr/bin/env python3
# Python 3.10
'''
Upload to dropbox
Required utilities:
dbxcli (v3.0.0) https://github.com/dropbox/dbxcli
Usage:
./upload.py <source_path> [dest_folder=/]
source_path ... path to local file or folder.
if this is a folder, the whole folder will be placed into the dest_folder on the dropbox
dest_folder ... path to dropbox folder. default is /, the root of the dropbox
TODOs:
* allow multiple source paths, or globs
* --skip=n argument to retry uploads, skipping a number of files
'''
RETRIES = -1 # 0 is no retries, -1 is indefinite retries
RETRY_SLEEP = [5, 50] # [0] .. step (is multiplied by retry count), [1] .. max. sleep time
import subprocess
import argparse
import os.path
import signal
import time
import datetime
import math
class COLORS:
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
END = '\033[0m'
def run_cmd(cmd, return_exitcode_only=True):
# os.system is simple, but will signals won't work
# shell allows to specify cmd as a single string, as opposed to a list of args
completed_process = subprocess.run(cmd, shell=True)
if return_exitcode_only: return completed_process.returncode
else: return completed_process
# check if target_folder exists and create if not
def make_dir(target_folder, silent = True, prefix = ''):
print(f'{prefix}Creating folder {target_folder}')
if target_folder == '/': return 0
redir = ' > /dev/null 2>&1' if silent else ''
code = run_cmd(f"dbxcli ls '{target_folder}'{redir}")
if code != 0:
code = run_cmd(f"dbxcli mkdir '{target_folder}'")
if code != 0:
print(f'Error creating folder (code {code})')
return code
return 0
def upload_file(src, dst, silent = True, prefix = ''):
print(f'{prefix}Uploading {src} -> {dst}')
redir = ' > /dev/null 2>&1' if silent else ''
return run_cmd(f"dbxcli put '{src}' '{dst}'{redir}")
def put(source_path, target_folder = '/', retry = RETRIES):
if not os.path.exists(source_path):
print(f'File/folder doesn\'t exists: {source_path}')
count = 0
ok = 0
fail = 0
fails = []
def print_stats(prefix = ''):
if (fail == 0):
print(f'{prefix}Total: {count} {COLORS.GREEN}Ok: {ok}{COLORS.END} Failed: {fail}')
else:
print(f'{prefix}Total: {count} {COLORS.GREEN}Ok: {ok}{COLORS.END} {COLORS.RED}Failed: {fail}{COLORS.END}')
if os.path.isdir(source_path):
# we want to copy the dir itself, append it to the target
if source_path.endswith('/'): source_path = source_path[:-1]
target_folder = os.path.join(target_folder, os.path.basename(source_path))
if os.path.isdir(source_path):
# upload contents of folder
for root, dirs, files in os.walk(source_path):
files.sort()
target_root = root[len(source_path):]
if target_root.startswith('/'): target_root = target_root[1:] # remove leading /, won't join otherwise
files = list( filter(lambda x: not x.startswith('.'), files) )
if len(files) > 0:
for file in files:
count += 1
if file.startswith('.'): continue
src = os.path.join(root, file)
dst = os.path.join(target_folder, target_root, file)
code = -1
tries = 0
while code != 0 and ( retry == -1 or tries < retry + 1 ):
if tries > 0:
sleep = min(RETRY_SLEEP[0] * tries, RETRY_SLEEP[1])
print(f' Retry {tries}/{retry if retry > 0 else "∞"}, waiting {sleep}s...')
time.sleep(sleep)
small = os.path.getsize(src) < 100 * 1_000_000 # 100 MB
code = upload_file(src, dst, silent=small, prefix=f'({count}) ')
tries += 1
if code == 0: ok += 1
else:
fail += 1
print(f' {COLORS.RED}FAILED ({code}){COLORS.END}')
fails.append(f"./upload.py '{src}' '{os.path.dirname(dst)}'")
print_stats()
else: # empty dir, create it
count += 1
dst = os.path.join(target_folder, target_root)
code = make_dir(dst, prefix=f'({count}) ')
if code == 0:
ok += 1
else:
fail += 1
print(f' {COLORS.RED}FAILED ({code}){COLORS.END}')
fails.append(f"EMPTY DIR: '{dst}'")
print_stats()
else:
# single file
src = source_path
dst = os.path.join( target_folder, os.path.basename(source_path) )
small = os.path.getsize(src) < 100 * 1_000_000 # 100 MB
code = upload_file(src, dst, silent=small)
count += 1
if code == 0:
ok += 1
else:
fail += 1
print(f' FAILED')
fails.append(f"./upload.py '{src}' '{os.path.dirname(dst)}'")
print_stats()
if len(fails) > 0:
print()
print('To retry fails, type:')
print('\n'.join(fails))
def print_elapsed():
global start_time
elapsed = datetime.timedelta(seconds = math.floor(time.time()-start_time) )
print(f'Elapsed time: {str(elapsed)}')
def signal_handler(sig, stack=None):
print(f'\nCaught signal {signal.Signals(sig).name}: Exiting')
print_elapsed()
exit(1)
if __name__ == '__main__':
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGQUIT, signal_handler)
parser = argparse.ArgumentParser()
parser.add_argument('source_path') # local source path. a file or a folder
parser.add_argument('dest_folder', nargs='?', default='/') # remote destination folder
args = parser.parse_args()
global start_time
start_time = time.time()
put(args.source_path, args.dest_folder)
print()
print_elapsed()