Skip to content

Commit

Permalink
Merge pull request #127 from qld-gov-au/enable-download-proxy
Browse files Browse the repository at this point in the history
Enable file downloads to go through a proxy if needed
  • Loading branch information
amercader authored Oct 28, 2021
2 parents 710e1de + 996af8e commit ef30095
Showing 1 changed file with 6 additions and 7 deletions.
13 changes: 6 additions & 7 deletions ckanext/xloader/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,13 +333,12 @@ def _download_resource_data(resource, data, api_key, logger):

def get_response(url, headers):
def get_url():
return requests.get(
url,
headers=headers,
timeout=DOWNLOAD_TIMEOUT,
verify=SSL_VERIFY,
stream=True, # just gets the headers for now
)
kwargs = {'headers': headers, 'timeout': DOWNLOAD_TIMEOUT,
'verify': SSL_VERIFY, 'stream': True} # just gets the headers for now
if 'ckan.download_proxy' in config:
proxy = config.get('ckan.download_proxy')
kwargs['proxies'] = {'http': proxy, 'https': proxy}
return requests.get(url, **kwargs)
response = get_url()
if response.status_code == 202:
# Seen: https://data-cdfw.opendata.arcgis.com/datasets
Expand Down

0 comments on commit ef30095

Please sign in to comment.