From e5fa4df5f9f37046dbada7215072822a39b6ea3b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 11:47:04 +0530 Subject: [PATCH 01/18] fix: Fetch env python if exists before returning fallback --- bench/bench.py | 6 ++++++ bench/utils/bench.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bench/bench.py b/bench/bench.py index d94c9f704..11ed67d46 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -6,6 +6,7 @@ import json import sys import logging +from glob import glob from typing import List, MutableSequence, TYPE_CHECKING, Union # imports - module imports @@ -71,6 +72,11 @@ def __init__(self, path): @property def python(self) -> str: + existing_python_bins = glob(f"{self.name}/env/bin/python*") + + if existing_python_bins: + return existing_python_bins[0] + return get_env_cmd("python", bench_path=self.name) @property diff --git a/bench/utils/bench.py b/bench/utils/bench.py index c02636578..999915add 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -40,7 +40,7 @@ def get_venv_path(verbose=False, python="python3"): if is_venv_installed: return f"{python} -m venv" else: - log("virtualenv cannot be found", level=2) + log("venv cannot be found", level=2) def update_node_packages(bench_path=".", apps=None): From 3b173f92c8e9889eb0419ab737ed966d0751d7d8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 13:59:25 +0530 Subject: [PATCH 02/18] ci: Use Ubuntu focal for tests --- .travis.yml | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 76618d5ad..1fa226713 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -dist: bionic +dist: focal sudo: true git: @@ -51,26 +51,18 @@ matrix: script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init install: - - python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip + - wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; - - if [ $TEST == "bench" ];then - wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; - sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; + - nvm install 14; + nvm use 14; - nvm install 14; - nvm use 14; + - mkdir -p ~/.bench; + cp -r $TRAVIS_BUILD_DIR/* ~/.bench; + python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; + python -m pip install -U -e ~/.bench; - mkdir -p ~/.bench; - cp -r $TRAVIS_BUILD_DIR/* ~/.bench; - python -m pip install -U -e ~/.bench; - - mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; - mysql -u root -e "FLUSH PRIVILEGES"; - fi - - - if [ $TEST == "easy_install" ];then - mkdir -p /tmp/.bench; - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench; - fi + - mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; + mysql -u root -e "FLUSH PRIVILEGES"; From 1a062263cf72b18fc4fa0aa3222aaf2a6c99bd87 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 14:15:23 +0530 Subject: [PATCH 03/18] ci: Bump to mariadb 10.6 --- .travis.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1fa226713..58a78742c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,48 +6,40 @@ git: depth: 1 addons: - mariadb: '10.3' + mariadb: '10.6' matrix: include: - name: "Python 3.7 Basic Setup" python: 3.7 - env: TEST=bench script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.8 Basic Setup" python: 3.8 - env: TEST=bench script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.9 Basic Setup" python: 3.9 - env: TEST=bench script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.10 Basic Setup" python: "3.10" - env: TEST=bench script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.7 Production Setup" python: 3.7 - env: TEST=bench script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.10 Production Setup" python: "3.10" - env: TEST=bench script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.7 Tests" python: 3.7 - env: TEST=bench script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init - name: "Python 3.10 Tests" python: "3.10" - env: TEST=bench script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init install: @@ -62,7 +54,7 @@ install: python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; python -m pip install -U -e ~/.bench; - - mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; - mysql -u root -e "FLUSH PRIVILEGES"; + - mariadb --host 127.0.0.1 --port 3306 -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; From e7c0c264d9669bb0441a2a2b782c0876b27a73b2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 14:25:04 +0530 Subject: [PATCH 04/18] ci: Migrate tests from Travis to GHA --- .github/workflows/ci.yml | 123 +++++++++++++++++++++++++++++++++++++++ .travis.yml | 60 ------------------- 2 files changed, 123 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..4c23f063b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,123 @@ +name: 'CI' + +on: + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +jobs: + base_setup: + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + matrix: + python-version: [ '3.7', '3.8', '3.9', '3.10' ] + + name: Base (${{ matrix.python-version }}) + + services: + mariadb: + image: mariadb:10.6 + env: + MARIADB_ROOT_PASSWORD: travis + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: | + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install redis-server + + python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; + python -m pip install -U -e ${GITHUB_WORKSPACE}; + + - run: python ${GITHUB_WORKSPACE}/bench/tests/test_init.py TestBenchInit.basic + + production_setup: + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + matrix: + python-version: [ '3.7', '3.10' ] + + name: Production (${{ matrix.python-version }}) + + services: + mariadb: + image: mariadb:10.6 + env: + MARIADB_ROOT_PASSWORD: travis + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: | + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install redis-server + + python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; + python -m pip install -U -e ${GITHUB_WORKSPACE}; + + - run: python bench/tests/test_setup_production.py TestSetupProduction.production + + tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: [base_setup] + + strategy: + matrix: + python-version: [ '3.7', '3.10' ] + + name: Tests (${{ matrix.python-version }}) + + services: + mariadb: + image: mariadb:10.6 + env: + MARIADB_ROOT_PASSWORD: travis + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: | + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; + sudo apt install redis-server + + python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; + python -m pip install -U -e ${GITHUB_WORKSPACE}; + + - run: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 58a78742c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,60 +0,0 @@ -language: python -dist: focal -sudo: true - -git: - depth: 1 - -addons: - mariadb: '10.6' - -matrix: - include: - - name: "Python 3.7 Basic Setup" - python: 3.7 - script: python bench/tests/test_init.py TestBenchInit.basic - - - name: "Python 3.8 Basic Setup" - python: 3.8 - script: python bench/tests/test_init.py TestBenchInit.basic - - - name: "Python 3.9 Basic Setup" - python: 3.9 - script: python bench/tests/test_init.py TestBenchInit.basic - - - name: "Python 3.10 Basic Setup" - python: "3.10" - script: python bench/tests/test_init.py TestBenchInit.basic - - - name: "Python 3.7 Production Setup" - python: 3.7 - script: python bench/tests/test_setup_production.py TestSetupProduction.production - - - name: "Python 3.10 Production Setup" - python: "3.10" - script: python bench/tests/test_setup_production.py TestSetupProduction.production - - - name: "Python 3.7 Tests" - python: 3.7 - script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init - - - name: "Python 3.10 Tests" - python: "3.10" - script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init - -install: - - wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; - sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb; - - - nvm install 14; - nvm use 14; - - - mkdir -p ~/.bench; - cp -r $TRAVIS_BUILD_DIR/* ~/.bench; - python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip; - python -m pip install -U -e ~/.bench; - - - mariadb --host 127.0.0.1 --port 3306 -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; From c9c6bf4512ba0c1c58df92a2384347bd77cc558d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 14:54:01 +0530 Subject: [PATCH 05/18] fix: Use hardcoded python3 env cmd for Bench.python --- bench/bench.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 11ed67d46..6b5dd3d1c 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -6,7 +6,6 @@ import json import sys import logging -from glob import glob from typing import List, MutableSequence, TYPE_CHECKING, Union # imports - module imports @@ -72,12 +71,7 @@ def __init__(self, path): @property def python(self) -> str: - existing_python_bins = glob(f"{self.name}/env/bin/python*") - - if existing_python_bins: - return existing_python_bins[0] - - return get_env_cmd("python", bench_path=self.name) + return get_env_cmd("python3", bench_path=self.name) @property def shallow_clone(self) -> bool: From 137c786f5ea48d8908045b96860a9a24551f3d71 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 15:16:15 +0530 Subject: [PATCH 06/18] ci: Add Bench linters action --- .github/workflows/linters.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/linters.yml diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 000000000..1a2ab3cf2 --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,39 @@ +name: Linters + +on: + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +jobs: + linter: + name: 'Pre Commit Hooks' + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 + with: + extra_args: --files 'bench/' + + deps-vulnerable-check: + name: 'Vulnerable Dependency Check' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + - name: 'Pip Audit' + run: | + pip install pip-audit + pip-audit ${GITHUB_WORKSPACE} From b4a6f7fea8c85e9b3ef502301b16da9f6a940b2f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 15:40:54 +0530 Subject: [PATCH 07/18] test(app_states): Set git committer info in config --- bench/tests/test_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bench/tests/test_utils.py b/bench/tests/test_utils.py index db830e1b0..e0137dcaf 100644 --- a/bench/tests/test_utils.py +++ b/bench/tests/test_utils.py @@ -75,6 +75,18 @@ def test_app_states(self): f.write("__version__ = '11.0'") subprocess.run(["git", "add", "."], cwd=frappe_path, capture_output=True, check=True) + subprocess.run( + ["git", "config", "user.email", "bench-test_app_states@gha.com"], + cwd=frappe_path, + capture_output=True, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "App States Test"], + cwd=frappe_path, + capture_output=True, + check=True, + ) subprocess.run( ["git", "commit", "-m", "temp"], cwd=frappe_path, capture_output=True, check=True ) From 73719a26e2a2bd3ee6376b1289bebd06700d615c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 16:10:42 +0530 Subject: [PATCH 08/18] test(init): Run exec_cmd without raise [bug] --- bench/tests/test_init.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 1d4e10273..f8b6e038e 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -143,14 +143,15 @@ def test_install_app(self): # create and install app on site self.new_site(site_name, bench_name) installed_app = not exec_cmd( - f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path + f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", + cwd=bench_path, + _raise=False, ) - app_installed_on_site = subprocess.check_output( - ["bench", "--site", site_name, "list-apps"], cwd=bench_path - ).decode("utf8") - if installed_app: + app_installed_on_site = subprocess.check_output( + ["bench", "--site", site_name, "list-apps"], cwd=bench_path + ).decode("utf8") self.assertTrue(TEST_FRAPPE_APP in app_installed_on_site) def test_remove_app(self): @@ -183,14 +184,18 @@ def test_switch_to_branch(self): prevoius_branch = f"version-{int(FRAPPE_BRANCH.split('-')[1]) - 1}" successful_switch = not exec_cmd( - f"bench switch-to-branch {prevoius_branch} frappe --upgrade", cwd=bench_path + f"bench switch-to-branch {prevoius_branch} frappe --upgrade", + cwd=bench_path, + _raise=False, ) if successful_switch: app_branch_after_switch = str(git.Repo(path=app_path).active_branch) self.assertEqual(prevoius_branch, app_branch_after_switch) successful_switch = not exec_cmd( - f"bench switch-to-branch {FRAPPE_BRANCH} frappe --upgrade", cwd=bench_path + f"bench switch-to-branch {FRAPPE_BRANCH} frappe --upgrade", + cwd=bench_path, + _raise=False, ) if successful_switch: app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) From 7f4f1b9dc688f2ce4fced7f68bc31ff4eccbd9bc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 16:11:32 +0530 Subject: [PATCH 09/18] ci: Remove needs cond for tests job --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c23f063b..1040df3a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,6 @@ jobs: tests: runs-on: ubuntu-latest timeout-minutes: 60 - needs: [base_setup] strategy: matrix: From bff9dfd00424421407f507330c62ba331339370f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 17:37:30 +0530 Subject: [PATCH 10/18] test: Skip asset building & resolving node deps --- bench/tests/test_init.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index f8b6e038e..622ffc87d 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -80,7 +80,6 @@ def test_new_site(self): site_config_path = os.path.join(site_path, "site_config.json") self.init_bench(bench_name) - exec_cmd("bench setup requirements --node", cwd=bench_path) self.new_site(site_name, bench_name) self.assertTrue(os.path.exists(site_path)) @@ -99,7 +98,7 @@ def test_new_site(self): def test_get_app(self): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path) + exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --skip-assets", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output( ["bench", "pip", "freeze"], cwd=bench_path @@ -111,7 +110,7 @@ def test_get_app_resolve_deps(self): FRAPPE_APP = "healthcare" self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps", cwd=bench_path) + exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps --skip-assets", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP))) states_path = os.path.join(bench_path, "sites", "apps.json") @@ -128,9 +127,9 @@ def test_install_app(self): bench_path = os.path.join(self.benches_path, "test-bench") self.init_bench(bench_name) - exec_cmd("bench setup requirements --node", cwd=bench_path) - exec_cmd("bench build", cwd=bench_path) - exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path) + exec_cmd( + f"bench get-app {TEST_FRAPPE_APP} --branch master --skip-assets", cwd=bench_path + ) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) @@ -158,9 +157,9 @@ def test_remove_app(self): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - exec_cmd("bench setup requirements --node", cwd=bench_path) exec_cmd( - f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path + f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite --skip-assets", + cwd=bench_path, ) exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path) From 01b77598ff2ff4cde4c75dad9b3a39526858c685 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Aug 2022 17:40:37 +0530 Subject: [PATCH 11/18] fix: Pass cwd for cmd subprocess execution Also, use python instead of python3 hardcoded in env --- bench/bench.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 6b5dd3d1c..3d286bf1f 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -71,7 +71,7 @@ def __init__(self, path): @property def python(self) -> str: - return get_env_cmd("python3", bench_path=self.name) + return get_env_cmd("python", bench_path=self.name) @property def shallow_clone(self) -> bool: @@ -352,15 +352,18 @@ def env(self, python="python3"): if not os.path.exists(self.bench.python): if virtualenv: - self.run(f"{virtualenv} {quiet_flag} env -p {python}") + self.run(f"{virtualenv} {quiet_flag} env -p {python}", cwd=self.bench.name) else: venv = get_venv_path(verbose=verbose, python=python) - self.run(f"{venv} env") + self.run(f"{venv} env", cwd=self.bench.name) self.pip() if os.path.exists(frappe): - self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}") + self.run( + f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}", + cwd=self.bench.name, + ) @step(title="Setting Up Bench Config", success="Bench Config Set Up") def config(self, redis=True, procfile=True): @@ -388,7 +391,9 @@ def pip(self, verbose=False): verbose = bench.cli.verbose or verbose quiet_flag = "" if verbose else "--quiet" - return self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip") + return self.run( + f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip", cwd=self.bench.name + ) def logging(self): from bench.utils import setup_logging From f60c2d0def62eb689aecd6f6174dc8dd7130b42f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 11:54:00 +0530 Subject: [PATCH 12/18] fix: Use venv module instead of virtualenv Only migrate-env requires virtualenv wrapper. However, it can be installed and run manually too. Virtualenv wrapper is patched in debian to change the path of bins - which venv is free from. --- bench/bench.py | 11 +++-------- bench/commands/setup.py | 2 +- bench/utils/__init__.py | 8 ++------ bench/utils/bench.py | 3 +++ docs/bench_usage.md | 2 +- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 3d286bf1f..a331e2707 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -30,7 +30,6 @@ restart_process_manager, remove_backups_crontab, get_venv_path, - get_virtualenv_path, get_env_cmd, ) from bench.utils.render import job, step @@ -71,7 +70,7 @@ def __init__(self, path): @property def python(self) -> str: - return get_env_cmd("python", bench_path=self.name) + return get_env_cmd("python*", bench_path=self.name) @property def shallow_clone(self) -> bool: @@ -347,15 +346,11 @@ def env(self, python="python3"): click.secho("Setting Up Environment", fg="yellow") frappe = os.path.join(self.bench.name, "apps", "frappe") - virtualenv = get_virtualenv_path(verbose=verbose) quiet_flag = "" if verbose else "--quiet" if not os.path.exists(self.bench.python): - if virtualenv: - self.run(f"{virtualenv} {quiet_flag} env -p {python}", cwd=self.bench.name) - else: - venv = get_venv_path(verbose=verbose, python=python) - self.run(f"{venv} env", cwd=self.bench.name) + venv = get_venv_path(verbose=verbose, python=python) + self.run(f"{venv} env", cwd=self.bench.name) self.pip() diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 4139e8fb2..4d1fa88ab 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -96,7 +96,7 @@ def setup_backups(): Bench(".").setup.backups() -@click.command("env", help="Setup virtualenv for bench") +@click.command("env", help="Setup Python environment for bench") @click.option( "--python", type=str, default="python3", help="Path to Python Executable." ) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 952ca2a45..778e5319a 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -234,7 +234,7 @@ def run_frappe_cmd(*args, **kwargs): f = get_env_cmd("python", bench_path=bench_path) sites_dir = os.path.join(bench_path, "sites") - is_async = False if from_command_line else True + is_async = not from_command_line if is_async: stderr = stdout = subprocess.PIPE else: @@ -247,11 +247,7 @@ def run_frappe_cmd(*args, **kwargs): stderr=stderr, ) - if is_async: - return_code = print_output(p) - else: - return_code = p.wait() - + return_code = print_output(p) if is_async else p.wait() if return_code > 0: sys.exit(return_code) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 999915add..f2644db75 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -172,6 +172,9 @@ def migrate_env(python, backup=False): path = os.getcwd() python = which(python) virtualenv = which("virtualenv") + if not virtualenv: + raise FileNotFoundError("`virtualenv` not found. Install it and try again.") + pvenv = os.path.join(path, nvenv) # Clear Cache before Bench Dies. diff --git a/docs/bench_usage.md b/docs/bench_usage.md index 7bf52a1a3..07b7e0802 100644 --- a/docs/bench_usage.md +++ b/docs/bench_usage.md @@ -130,7 +130,7 @@ The setup commands used for setting up the Frappe environment in context of the - **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password - - **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory. + - **env**: Setup Python virtual environment for bench. This sets up a `env` folder under the root of the bench directory. - **redis**: Generates configuration for Redis - **fonts**: Add Frappe fonts to system - **config**: Generate or over-write sites/common_site_config.json From b37135e5b19888ef6bae5a79cb6d7bc0545e3097 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 11:57:38 +0530 Subject: [PATCH 13/18] refactor: get_env_cmd * Use globbing first to identify cmd in env paths * Unbound cache which clears on chdir * Allow passing patterns like 'python*' as cmd to match --- bench/cli.py | 1 + bench/utils/bench.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index b38aea306..d01b6eb2a 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -245,6 +245,7 @@ def setup_clear_cache(): def _chdir(*args, **kwargs): Bench.cache_clear() + get_env_cmd.cache_clear() return f(*args, **kwargs) os.chdir = _chdir diff --git a/bench/utils/bench.py b/bench/utils/bench.py index f2644db75..a3a0d1407 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -6,6 +6,8 @@ import re import subprocess import sys +from functools import lru_cache +from glob import glob from json.decoder import JSONDecodeError # imports - third party imports @@ -19,17 +21,18 @@ logger = logging.getLogger(bench.PROJECT_NAME) +@lru_cache(maxsize=None) def get_env_cmd(cmd, bench_path="."): - return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) - + # this supports envs' generated by patched virtualenv or venv (which may cause an extra 'local' folder to be created) -def get_virtualenv_path(verbose=False): - virtualenv_path = which("virtualenv") + existing_python_bins = glob( + os.path.abspath(os.path.join(bench_path, "env", "**", "bin", cmd)), recursive=True + ) - if not virtualenv_path and verbose: - log("virtualenv cannot be found", level=2) + if existing_python_bins: + return existing_python_bins[0] - return virtualenv_path + return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) def get_venv_path(verbose=False, python="python3"): From 24b9af605b08849ee506677a6c812c1e87ea3326 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 11:59:34 +0530 Subject: [PATCH 14/18] fix: Use python* to match any pattern in env --- bench/cli.py | 6 +++--- bench/commands/make.py | 2 +- bench/utils/__init__.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index d01b6eb2a..e2e462fad 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -190,13 +190,13 @@ def change_uid(): def app_cmd(bench_path="."): - f = get_env_cmd("python", bench_path=bench_path) + f = get_env_cmd("python*", bench_path=bench_path) os.chdir(os.path.join(bench_path, "sites")) os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:]) def frappe_cmd(bench_path="."): - f = get_env_cmd("python", bench_path=bench_path) + f = get_env_cmd("python*", bench_path=bench_path) os.chdir(os.path.join(bench_path, "sites")) os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:]) @@ -216,7 +216,7 @@ def get_frappe_commands(): def get_frappe_help(bench_path="."): - python = get_env_cmd("python", bench_path=bench_path) + python = get_env_cmd("python*", bench_path=bench_path) sites_path = os.path.join(bench_path, "sites") try: out = get_cmd_output( diff --git a/bench/commands/make.py b/bench/commands/make.py index 42ce08cd6..2895f79d8 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -227,5 +227,5 @@ def pip(ctx, args): from bench.utils.bench import get_env_cmd - env_py = get_env_cmd("python") + env_py = get_env_cmd("python*") os.execv(env_py, (env_py, "-m", "pip") + args) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 778e5319a..9a14920fc 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -231,7 +231,7 @@ def run_frappe_cmd(*args, **kwargs): from bench.utils.bench import get_env_cmd bench_path = kwargs.get("bench_path", ".") - f = get_env_cmd("python", bench_path=bench_path) + f = get_env_cmd("python*", bench_path=bench_path) sites_dir = os.path.join(bench_path, "sites") is_async = not from_command_line @@ -386,7 +386,7 @@ def generate_command_cache(bench_path=".") -> List: """ from bench.utils.bench import get_env_cmd - python = get_env_cmd("python", bench_path=bench_path) + python = get_env_cmd("python*", bench_path=bench_path) sites_path = os.path.join(bench_path, "sites") if os.path.exists(bench_cache_file): From 78742b9546dde7f3910ce25cc03f0f9288c0ed3e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 12:00:08 +0530 Subject: [PATCH 15/18] refactor: Use specific lru_cache imports over entire module's --- bench/app.py | 4 ++-- bench/bench.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/app.py b/bench/app.py index 6406260ec..aae317a36 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,5 +1,5 @@ # imports - standard imports -import functools +from functools import lru_cache import json import logging import os @@ -149,7 +149,7 @@ def get_ssh_url(self): return f"git@{self.remote_server}:{self.org}/{self.repo}.git" -@functools.lru_cache(maxsize=None) +@lru_cache(maxsize=None) class App(AppMeta): def __init__( self, diff --git a/bench/bench.py b/bench/bench.py index a331e2707..8015d0a36 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -1,6 +1,6 @@ # imports - standard imports import subprocess -import functools +from functools import lru_cache import os import shutil import json @@ -54,7 +54,7 @@ def validate_app_uninstall(self, app): validate_app_installed_on_sites(app, bench_path=self.name) -@functools.lru_cache(maxsize=None) +@lru_cache(maxsize=None) class Bench(Base, Validator): def __init__(self, path): self.name = path From 6ae1997bffb3d75083fb5c720336d036b18b22ce Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 12:18:40 +0530 Subject: [PATCH 16/18] fix(utils): Strip * from cmd via get_env_cmd --- bench/utils/bench.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index a3a0d1407..98cfec2ee 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -22,17 +22,18 @@ @lru_cache(maxsize=None) -def get_env_cmd(cmd, bench_path="."): +def get_env_cmd(cmd: str, bench_path: str = ".") -> str: # this supports envs' generated by patched virtualenv or venv (which may cause an extra 'local' folder to be created) existing_python_bins = glob( - os.path.abspath(os.path.join(bench_path, "env", "**", "bin", cmd)), recursive=True + os.path.join(bench_path, "env", "**", "bin", cmd), recursive=True ) if existing_python_bins: return existing_python_bins[0] - return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) + cmd = cmd.strip("*") + return os.path.join(bench_path, "env", "bin", cmd) def get_venv_path(verbose=False, python="python3"): From 10473b60076dde4080ce28222031a332ee7d313a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 12:46:03 +0530 Subject: [PATCH 17/18] fix: Pass abs path from get_env_cmd --- bench/utils/bench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 98cfec2ee..cf590e6f1 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -30,10 +30,10 @@ def get_env_cmd(cmd: str, bench_path: str = ".") -> str: ) if existing_python_bins: - return existing_python_bins[0] + return os.path.abspath(existing_python_bins[0]) cmd = cmd.strip("*") - return os.path.join(bench_path, "env", "bin", cmd) + return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) def get_venv_path(verbose=False, python="python3"): From f77352966b74defd053c2d73bb3fe942c4559eed Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 2 Aug 2022 13:14:55 +0530 Subject: [PATCH 18/18] fix: Remove frappe cmd caching in .bench.cmd --- bench/cli.py | 13 ++----------- bench/commands/__init__.py | 4 ---- bench/commands/utils.py | 14 -------------- bench/utils/__init__.py | 26 +++++--------------------- 4 files changed, 7 insertions(+), 50 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index b38aea306..797bc69d2 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,7 +1,6 @@ # imports - standard imports import atexit from contextlib import contextmanager -import json from logging import Logger import os import pwd @@ -16,11 +15,10 @@ from bench.commands import bench_command from bench.config.common_site_config import get_config from bench.utils import ( - bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, - generate_command_cache, + get_env_frappe_commands, get_cmd_output, is_bench_directory, is_dist_editable, @@ -201,18 +199,11 @@ def frappe_cmd(bench_path="."): os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:]) -def get_cached_frappe_commands(): - if os.path.exists(bench_cache_file): - command_dump = open(bench_cache_file).read() or "[]" - return set(json.loads(command_dump)) - return set() - - def get_frappe_commands(): if not is_bench_directory(): return set() - return set(generate_command_cache()) + return set(get_env_frappe_commands()) def get_frappe_help(bench_path="."): diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index d26ffecde..5ef142121 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -74,11 +74,9 @@ def bench_command(bench_path="."): from bench.commands.utils import ( backup_all_sites, bench_src, - clear_command_cache, disable_production, download_translations, find_benches, - generate_command_cache, migrate_env, renew_lets_encrypt, restart, @@ -110,8 +108,6 @@ def bench_command(bench_path="."): bench_command.add_command(bench_src) bench_command.add_command(find_benches) bench_command.add_command(migrate_env) -bench_command.add_command(generate_command_cache) -bench_command.add_command(clear_command_cache) from bench.commands.setup import setup diff --git a/bench/commands/utils.py b/bench/commands/utils.py index d606788d3..9882e8f01 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -176,17 +176,3 @@ def migrate_env(python, backup=True): from bench.utils.bench import migrate_env migrate_env(python=python, backup=backup) - - -@click.command("generate-command-cache", help="Caches Frappe Framework commands") -def generate_command_cache(bench_path="."): - from bench.utils import generate_command_cache - - return generate_command_cache(bench_path=bench_path) - - -@click.command("clear-command-cache", help="Clears Frappe Framework cached commands") -def clear_command_cache(bench_path="."): - from bench.utils import clear_command_cache - - return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 952ca2a45..bfd7ffdee 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -22,7 +22,6 @@ ) logger = logging.getLogger(PROJECT_NAME) -bench_cache_file = ".bench.cmd" paths_in_app = ("hooks.py", "modules.txt", "patches.txt") paths_in_bench = ("apps", "sites", "config", "logs", "config/pids") sudoers_file = "/etc/sudoers.d/frappe" @@ -384,7 +383,7 @@ def find_parent_bench(path: str) -> str: return find_parent_bench(parent_dir) -def generate_command_cache(bench_path=".") -> List: +def get_env_frappe_commands(bench_path=".") -> List: """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ @@ -393,16 +392,12 @@ def generate_command_cache(bench_path=".") -> List: python = get_env_cmd("python", bench_path=bench_path) sites_path = os.path.join(bench_path, "sites") - if os.path.exists(bench_cache_file): - os.remove(bench_cache_file) - try: - output = get_cmd_output( - f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path + return json.loads( + get_cmd_output( + f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path + ) ) - with open(bench_cache_file, "w") as f: - json.dump(eval(output), f) - return json.loads(output) except subprocess.CalledProcessError as e: if hasattr(e, "stderr"): @@ -411,17 +406,6 @@ def generate_command_cache(bench_path=".") -> List: return [] -def clear_command_cache(bench_path="."): - """Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - - if os.path.exists(bench_cache_file): - os.remove(bench_cache_file) - else: - print("Bench command cache doesn't exist in this folder!") - - def find_org(org_repo): import requests