Merge pull request #3788 from Flamefire/python27

clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x

.github/workflows/unit_tests.yml

@@ -101,12 +101,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -191,7 +191,17 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

easybuild/framework/easyconfig/easyconfig.py

@@ -46,6 +46,7 @@
 import functools
 import os
 import re
+from collections import OrderedDict
 from contextlib import contextmanager
 
 import easybuild.tools.filetools as filetools
@@ -74,7 +75,7 @@
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
 from easybuild.tools.modules import modules_tool, NoModulesTool
-from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
+from easybuild.tools.py2vs3 import create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA

easybuild/framework/easyconfig/tools.py

@@ -45,6 +45,7 @@
 import re
 import sys
 import tempfile
+from collections import OrderedDict
 
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
@@ -64,7 +65,6 @@
 from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
 from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr
 from easybuild.tools.multidiff import multidiff
-from easybuild.tools.py2vs3 import OrderedDict
 from easybuild.tools.toolchain.toolchain import is_system_toolchain
 from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.utilities import only_if_module_is_available, quote_str

easybuild/tools/build_details.py

@@ -31,8 +31,8 @@
 * Stijn De Weirdt (Ghent University)
 """
 import time
+from collections import OrderedDict
 from easybuild.tools.filetools import det_size
-from easybuild.tools.py2vs3 import OrderedDict
 from easybuild.tools.systemtools import get_system_info
 from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION
 

easybuild/tools/docs.py

@@ -40,6 +40,7 @@
 import inspect
 import json
 import os
+from collections import OrderedDict
 from easybuild.tools import LooseVersion
 
 from easybuild.base import fancylogger
@@ -60,7 +61,7 @@
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import read_file
 from easybuild.tools.modules import modules_tool
-from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase
+from easybuild.tools.py2vs3 import ascii_lowercase
 from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES

easybuild/tools/options.py

@@ -45,6 +45,7 @@
 import sys
 import tempfile
 import pwd
+from collections import OrderedDict
 
 import easybuild.tools.environment as env
 from easybuild.base import fancylogger  # build_log should always stay there, to ensure EasyBuildLog
@@ -97,7 +98,7 @@
 from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
 from easybuild.tools.modules import Lmod
-from easybuild.tools.py2vs3 import OrderedDict, string_type
+from easybuild.tools.py2vs3 import string_type
 from easybuild.tools.robot import det_robot_path
 from easybuild.tools.run import run_cmd
 from easybuild.tools.package.utilities import avail_package_naming_schemes

easybuild/tools/output.py

@@ -32,10 +32,10 @@
 * Jørgen Nordmoen (University of Oslo)
 """
 import functools
+from collections import OrderedDict
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
-from easybuild.tools.py2vs3 import OrderedDict
 
 try:
     from rich.console import Console, Group

easybuild/tools/py2vs3/py2.py

@@ -52,7 +52,7 @@
 
 
 # reload function (built-in in Python 2)
-reload = reload
+reload = reload  # noqa: F821
 
 # string type that can be used in 'isinstance' calls
 string_type = basestring
@@ -89,9 +89,12 @@ def subprocess_terminate(proc, timeout):
         proc.terminate()
 
 
+# Wrapped in exec to avoid invalid syntax warnings for Python 3
+exec('''
 def raise_with_traceback(exception_class, message, traceback):
     """Raise exception of specified class with given message and traceback."""
     raise exception_class, message, traceback  # noqa: E999
+''')
 
 
 def extract_method_name(method_func):

easybuild/tools/systemtools.py

@@ -42,6 +42,7 @@
 import sys
 import termios
 import warnings
+from collections import OrderedDict
 from ctypes.util import find_library
 from socket import gethostname
 from easybuild.tools.py2vs3 import subprocess_popen_text
@@ -64,7 +65,7 @@
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import IGNORE
 from easybuild.tools.filetools import is_readable, read_file, which
-from easybuild.tools.py2vs3 import OrderedDict, string_type
+from easybuild.tools.py2vs3 import string_type
 from easybuild.tools.run import run_cmd
 
 
@@ -735,7 +736,6 @@ def get_os_name():
     # platform.linux_distribution was removed in Python 3.8,
     # see https://docs.python.org/2/library/platform.html#platform.linux_distribution
     if hasattr(platform, 'linux_distribution'):
-        # platform.linux_distribution is more useful, but only available since Python 2.6
         # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS)
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=PendingDeprecationWarning)

eb

@@ -40,8 +40,8 @@ keyboard_interrupt() {
 
 trap keyboard_interrupt SIGINT
 
-# Python 2.6+ or 3.5+ required
-REQ_MIN_PY2VER=6
+# Python 2.7+ or 3.5+ required
+REQ_MIN_PY2VER=7
 REQ_MIN_PY3VER=5
 
 EASYBUILD_MAIN='easybuild.main'

requirements.txt

@@ -1,10 +1,12 @@
 # keyring is required to provide GitHub token to EasyBuild;
 # for recent versions of keyring, keyrings.alt must be installed too
-keyring
+# 19.0 dropped Python 2 support
+keyring<19.0; python_version < '3.0'
+keyring; python_version >= '3.0'
 keyrings.alt
 
 # GitPython 3.1.15 deprecates Python 3.5
-GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6'
+GitPython<3.1.15; python_version >= '3.0' and python_version < '3.6'
 GitPython; python_version >= '3.6' or python_version <= '3.0'
 
 # autopep8

test/framework/easyconfig.py

@@ -38,6 +38,7 @@
 import sys
 import tempfile
 import textwrap
+from collections import OrderedDict
 from easybuild.tools import LooseVersion
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
@@ -72,7 +73,7 @@
 from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.options import parse_external_modules_metadata
-from easybuild.tools.py2vs3 import OrderedDict, reload
+from easybuild.tools.py2vs3 import reload
 from easybuild.tools.robot import det_robot_path, resolve_dependencies
 from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64
 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version

test/framework/repository.py

@@ -41,7 +41,6 @@
 from easybuild.tools.filetools import read_file
 from easybuild.tools.repository.filerepo import FileRepository
 from easybuild.tools.repository.gitrepo import GitRepository
-from easybuild.tools.repository.hgrepo import HgRepository
 from easybuild.tools.repository.svnrepo import SvnRepository
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.run import run_cmd
@@ -129,27 +128,6 @@ def test_svnrepo(self):
         self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md'))
         shutil.rmtree(repo.wc)
 
-    # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1
-    # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config
-    # (which implies dropping support for Python 2.6)
-    # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678
-    def DISABLED_test_hgrepo(self):
-        """Test using HgRepository."""
-        # only run this test if pysvn Python module is available
-        try:
-            import hglib  # noqa
-        except ImportError:
-            print("(skipping HgRepository test)")
-            return
-
-        # GitHub also supports SVN
-        test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository'
-
-        repo = HgRepository(test_repo_url)
-        repo.init()
-        self.assertExists(os.path.join(repo.wc, 'README'))
-        shutil.rmtree(repo.wc)
-
     def test_init_repository(self):
         """Test use of init_repository function."""
         repo = init_repository('FileRepository', self.path)