Skip to content

Commit

Permalink
bodsdata.py: Copy file if local
Browse files Browse the repository at this point in the history
  • Loading branch information
Ed (ODSC) committed Feb 24, 2024
1 parent 0fcbc93 commit 026fdfd
Showing 1 changed file with 11 additions and 0 deletions.
11 changes: 11 additions & 0 deletions bodsdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,15 @@ def download_file(url, source, name=None):
zipObj.extractall(path=f'{output_dir}/{source}_download/')
os.remove(filename)

def copy_file(path, source, name=None):
print('Copying File')
os.makedirs(f'{output_dir}/{source}_download', exist_ok=True)
if not name:
name = path.split('/')[-1]

filename = f'{output_dir}/{source}_download/{name}'

shutil.copy(path, filename)

def download_files_s3(source, s3_path_pattern, latest=False, bucket="bodsdata-oo", sample=None):
""" Download file to form s3 with given regex pattern.
Expand Down Expand Up @@ -1044,6 +1053,8 @@ def run_pipeline(source, title, description, download, upload, bucket = '', chec
remove_download(source)
if bucket != '':
download_files_s3(s3_path_pattern=download, source=source, latest=False, bucket=bucket)
elif not ":" in download:
copy_file(download, source=source)
else:
download_file(download, source=source)
if check: check_data_consistency(source, check_missing_fields=check_missing_fields,
Expand Down

0 comments on commit 026fdfd

Please sign in to comment.