Add an --extra-source-urls CLI option to fetch sources from additiona…

…l URLs.

Add the --extra-source-urls CLI option, a comma separated list of
URLs that EasyBuild will fetch sources from. It replaces the hard-coded
EASYBUILD_SOURCES_URL, but keeps it as a default value. Also add --extra-source-urls-priority
CLI option is used to determine if these URLs are to be queried first or last. Possible
options are "primary" and "backup".

easybuild/framework/easyblock.py

@@ -69,6 +69,7 @@
 from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import EXTRA_SOURCE_URLS_PRIMARY
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
@@ -103,8 +104,6 @@
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
 
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
 DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -821,8 +820,14 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     source_urls = []
                 source_urls.extend(self.cfg['source_urls'])
 
-                # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+                # add extra-source-urls CLI as either a first check, or a fallback.
+                for url in build_option("extra_source_urls"):
+                    index = (
+                        0
+                        if build_option("extra_source_urls_priority") == EXTRA_SOURCE_URLS_PRIMARY
+                        else len(source_urls)
+                    )
+                    source_urls.insert(index, url + "/" + os.path.join(name_letter, location))
 
                 mkdir(targetdir, parents=True)
 

easybuild/tools/config.py

@@ -113,9 +113,15 @@
 DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders'
 DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
 DEFAULT_REPOSITORY = 'FileRepository'
+DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io',)
 DEFAULT_WAIT_ON_LOCK_INTERVAL = 60
 DEFAULT_WAIT_ON_LOCK_LIMIT = 0
 
+EXTRA_SOURCE_URLS_PRIMARY = 'primary'
+EXTRA_SOURCE_URLS_BACKUP = 'backup'
+EXTRA_SOURCE_URLS_PRIORITY_CHOICES = [EXTRA_SOURCE_URLS_PRIMARY, EXTRA_SOURCE_URLS_BACKUP]
+DEFAULT_EXTRA_SOURCE_URLS_PRIORITY = EXTRA_SOURCE_URLS_BACKUP
+
 EBROOT_ENV_VAR_ACTIONS = [ERROR, IGNORE, UNSET, WARN]
 LOADED_MODULES_ACTIONS = [ERROR, IGNORE, PURGE, UNLOAD, WARN]
 DEFAULT_ALLOW_LOADED_MODULES = ('EasyBuild',)
@@ -191,6 +197,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'download_timeout',
         'dump_test_report',
         'easyblock',
+        'extra_source_urls_priority',
         'envvars_user_modules',
         'extra_modules',
         'filter_deps',
@@ -357,6 +364,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     'defaultopt': [
         'default_opt_level',
     ],
+    DEFAULT_EXTRA_SOURCE_URLS: [
+        'extra_source_urls',
+    ],
     DEFAULT_ALLOW_LOADED_MODULES: [
         'allow_loaded_modules',
     ],

easybuild/tools/options.py

@@ -65,9 +65,11 @@
 from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
 from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
 from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT
+from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
+from easybuild.tools.config import DEFAULT_EXTRA_SOURCE_URLS_PRIORITY, DEFAULT_PR_TARGET_ACCOUNT
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
-from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
+from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, EXTRA_SOURCE_URLS_PRIORITY_CHOICES
+from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
 from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
@@ -394,6 +396,11 @@ def override_options(self):
                              None, 'store_true', False),
             'extra-modules': ("List of extra modules to load after setting up the build environment",
                               'strlist', 'extend', None),
+            "extra-source-urls": ("Specify different URLs to fetch sources from", "strlist", "store",
+                                  DEFAULT_EXTRA_SOURCE_URLS),
+            "extra-source-urls-priority": ("Specify whether sources-url should be tried before or after the "
+                                           "ones defined in the easyconfigs", "choice", "store_or_None",
+                                           DEFAULT_EXTRA_SOURCE_URLS_PRIORITY, EXTRA_SOURCE_URLS_PRIORITY_CHOICES),
             'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
                       "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
             'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "