From 965e178e8329c3af3defed984f79354d5a7ffbd7 Mon Sep 17 00:00:00 2001 From: Ameen Ahmed Date: Thu, 17 Nov 2022 16:26:31 +0300 Subject: [PATCH 01/54] fix: FileNotFound bug (#1383) --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 0a13d4bd8..07eb4799e 100755 --- a/bench/app.py +++ b/bench/app.py @@ -198,7 +198,7 @@ def get(self): @step(title="Archiving App {repo}", success="App {repo} Archived") def remove(self, no_backup: bool = False): - active_app_path = os.path.join("apps", self.name) + active_app_path = os.path.join("apps", self.repo) if no_backup: if not os.path.islink(active_app_path): From b57838f36615b659930a4f38cfd14375097a8ad1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 22 Nov 2022 12:16:54 +0530 Subject: [PATCH 02/54] feat: add `max_requests` to gunicorn args As gunicorn is long running process, potentially running for days without restart the workers might start accumulating garbage that's never cleaned up and memory usage spikes after some use. This largely happens because of third-party module imports like pandas, openpyxl, numpy etc. All of these are required only for few requests and can be easily re-loaded when required. `max_requests` restarts the worker after processing number of configured requests. How to use? - If you have more than 1 gunicorn workers then this is automatically enabled. You can tweak the max_requests parameter with `gunicorn_max_requests` key in common_site_config - If you just have 1 gunicorn worker (not recommended) then this is not automatically enabled as restarting the only worker can cause spikes in response times whenever restart is triggered. --- bench/config/common_site_config.py | 14 ++++++++++++++ bench/config/supervisor.py | 11 +++++++---- bench/config/systemd.py | 11 +++++++---- bench/config/templates/supervisor.conf | 2 +- .../systemd/frappe-bench-frappe-web.service | 2 +- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index d278dedbc..41242aad3 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -15,6 +15,7 @@ "live_reload": True, } +DEFAULT_MAX_REQUESTS = 5000 def setup_config(bench_path): make_pid_folder(bench_path) @@ -61,6 +62,19 @@ def get_gunicorn_workers(): return {"gunicorn_workers": multiprocessing.cpu_count() * 2 + 1} +def compute_max_requests_jitter(max_requests: int) -> int: + return int(max_requests * 0.1) + +def get_default_max_requests(worker_count: int): + """Get max requests and jitter config based on number of available workers.""" + + if worker_count <= 1: + # If there's only one worker then random restart can cause spikes in response times and + # can be annoying. Hence not enabled by default. + return 0 + return DEFAULT_MAX_REQUESTS + + def update_config_for_frappe(config, bench_path): ports = make_ports(bench_path) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index d38da668d..58015b81e 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,7 +8,7 @@ from bench.app import use_rq from bench.utils import get_bench_name, which from bench.bench import Bench -from bench.config.common_site_config import update_config, get_gunicorn_workers +from bench.config.common_site_config import update_config, get_gunicorn_workers, get_default_max_requests, compute_max_requests_jitter # imports - third party imports import click @@ -26,6 +26,9 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals template = bench.config.env().get_template("supervisor.conf") bench_dir = os.path.abspath(bench_path) + web_worker_count = config.get("gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"]) + max_requests = config.get("gunicorn_max_requests", get_default_max_requests(web_worker_count)) + config = template.render( **{ "bench_dir": bench_dir, @@ -39,9 +42,9 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"), "redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"), "webserver_port": config.get("webserver_port", 8000), - "gunicorn_workers": config.get( - "gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"] - ), + "gunicorn_workers": web_worker_count, + "gunicorn_max_requests": max_requests, + "gunicorn_max_requests_jitter": compute_max_requests_jitter(max_requests), "bench_name": get_bench_name(bench_path), "background_workers": config.get("background_workers") or 1, "bench_cmd": which("bench"), diff --git a/bench/config/systemd.py b/bench/config/systemd.py index d30edfc94..a677391cd 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -9,7 +9,7 @@ import bench from bench.app import use_rq from bench.bench import Bench -from bench.config.common_site_config import get_gunicorn_workers, update_config +from bench.config.common_site_config import get_gunicorn_workers, update_config, get_default_max_requests, compute_max_requests_jitter from bench.utils import exec_cmd, which, get_bench_name @@ -61,6 +61,9 @@ def generate_systemd_config( get_bench_name(bench_path) + "-frappe-long-worker@" + str(i + 1) + ".service" ) + web_worker_count = config.get("gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"]) + max_requests = config.get("gunicorn_max_requests", get_default_max_requests(web_worker_count)) + bench_info = { "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, "sites"), @@ -73,9 +76,9 @@ def generate_systemd_config( "redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"), "redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"), "webserver_port": config.get("webserver_port", 8000), - "gunicorn_workers": config.get( - "gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"] - ), + "gunicorn_workers": web_worker_count, + "gunicorn_max_requests": max_requests, + "gunicorn_max_requests_jitter": compute_max_requests_jitter(max_requests), "bench_name": get_bench_name(bench_path), "worker_target_wants": " ".join(background_workers), "bench_cmd": which("bench"), diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index 085cc2cf4..f29c1673e 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -3,7 +3,7 @@ ; killasgroup=true --> send kill signal to child processes too [program:{{ bench_name }}-frappe-web] -command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload +command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} --max-requests {{ gunicorn_max_requests }} --max-requests-jitter {{ gunicorn_max_requests_jitter }} -t {{ http_timeout }} frappe.app:application --preload priority=4 autostart=true autorestart=true diff --git a/bench/config/templates/systemd/frappe-bench-frappe-web.service b/bench/config/templates/systemd/frappe-bench-frappe-web.service index bb2f0e381..0621e1d34 100644 --- a/bench/config/templates/systemd/frappe-bench-frappe-web.service +++ b/bench/config/templates/systemd/frappe-bench-frappe-web.service @@ -6,7 +6,7 @@ PartOf={{ bench_name }}-web.target User={{ user }} Group={{ user }} Restart=always -ExecStart={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload +ExecStart={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} --max-requests {{ gunicorn_max_requests }} --max-requests-jitter {{ gunicorn_max_requests_jitter }} frappe.app:application --preload StandardOutput=file:{{ bench_dir }}/logs/web.log StandardError=file:{{ bench_dir }}/logs/web.error.log WorkingDirectory={{ sites_dir }} From d3cb7eceb40bbfda86382fee70e0b58010dd7d05 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 23 Nov 2022 11:47:09 +0530 Subject: [PATCH 03/54] ci: fix flake8 url --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b40aa0883..d608bf4a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,8 +29,8 @@ repos: - id: black additional_dependencies: ['click==8.0.4'] - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: ['flake8-bugbear',] From f45db01d9aaff6ab19e3cf9eeed6c2708313c957 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 23 Nov 2022 11:54:58 +0530 Subject: [PATCH 04/54] chore: remove deprecated easy install script --- docs/easy_install.md | 93 -------- install.py | 499 ------------------------------------------- 2 files changed, 592 deletions(-) delete mode 100644 docs/easy_install.md delete mode 100644 install.py diff --git a/docs/easy_install.md b/docs/easy_install.md deleted file mode 100644 index 91ee55666..000000000 --- a/docs/easy_install.md +++ /dev/null @@ -1,93 +0,0 @@ -# Easy Install Script - -- This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)` -- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwords.txt` -- MariaDB (root) password may be `password` on a fresh server -- You can then login as **Administrator** with the Administrator password -- The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. -- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com/tags/installation_problem) under the "Install / Update" category. - ---- - -## What will this script do? - -- Install all the pre-requisites -- Install the command line `bench` (under ~/.bench) -- Create a new bench (a folder that will contain your entire frappe/erpnext setup at ~/frappe-bench) -- Create a new ERPNext site on the bench (site1.local) - ---- - -## Getting started with easy install... - -Open your Terminal and enter: - -#### 0. Setup user & Download the install script - -If you are on a fresh server and logged in as root, at first create a dedicated user for frappe -& equip this user with sudo privileges - -``` - adduser [frappe-user] - usermod -aG sudo [frappe-user] -``` - -*(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* - -*(you can specify the flag --home to specify a directory for your [frappe-user]. Bench will follow the home directory specified by the user's home directory e.g. /data/[frappe-user]/frappe-bench)* - -Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup - - wget https://raw.githubusercontent.com/frappe/bench/develop/install.py - - -#### 1. Run the install script - - sudo python3 install.py - -*Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)* - -For production or development, append the `--production` or `--develop` flag to the command respectively. - - sudo python3 install.py --production --user [frappe-user] - -or - - sudo python3 install.py --develop - sudo python3 install.py --develop --user [frappe-user] - - sudo python3 install.py --production --user [frappe-user] --container - -*Note: `container` flag to install inside a container (this will prevent the `/proc/sys/vm/swappiness: Read-only` file system error)* - - - python3 install.py --production --version 11 --user [frappe-user] - -use --version flag to install specific version - - python3 install.py --production --version 11 --python python2.7 --user [frappe-user] - -use --python flag to specify virtual environments python version, by default script setup python3 - ---- - -## How do I start ERPNext - -1. For development: Go to your bench folder (`~[frappe-user]/frappe-bench` by default) and start the bench with `bench start` -2. For production: Your process will be setup and managed by `nginx` and `supervisor`. Checkout [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) for more information. - ---- - -## An error occured mid installation? - -TLDR; Save the logs! - -1. The easy install script starts multiple processes to install prerequisites, system dependencies, requirements, sets up locales, configuration files, etc. - -2. The script pipes all these process outputs and saves it under `/tmp/log/{easy-install-filename}.log` as prompted by the script in the beginning of the script or/and if something went wrong again. - -3. Retain this log file and share it in case you need help with proceeding with the install. Since, the file's saved under `/tmp` it'll be cleared by the system after a reboot. Be careful to save it elsewhere if needed! - -3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies. - -4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the script better! diff --git a/install.py b/install.py deleted file mode 100644 index 54248b9cf..000000000 --- a/install.py +++ /dev/null @@ -1,499 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import print_function - -import os -import sys -import subprocess -import getpass -import json -import multiprocessing -import shutil -import platform -import warnings -import datetime - - -tmp_bench_repo = os.path.join('/', 'tmp', '.bench') -tmp_log_folder = os.path.join('/', 'tmp', 'logs') -execution_timestamp = datetime.datetime.utcnow() -execution_day = "{:%Y-%m-%d}".format(execution_timestamp) -execution_time = "{:%H:%M}".format(execution_timestamp) -log_file_name = "easy-install__{0}__{1}.log".format(execution_day, execution_time.replace(':', '-')) -log_path = os.path.join(tmp_log_folder, log_file_name) -log_stream = sys.stdout -distro_required = not ((sys.version_info.major < 3) or (sys.version_info.major == 3 and sys.version_info.minor < 5)) - - -def log(message, level=0): - levels = { - 0: '\033[94m', # normal - 1: '\033[92m', # success - 2: '\033[91m', # fail - 3: '\033[93m' # warn/suggest - } - start = levels.get(level) or '' - end = '\033[0m' - print(start + message + end) - - -def setup_log_stream(args): - global log_stream - sys.stderr = sys.stdout - - if not args.verbose: - if not os.path.exists(tmp_log_folder): - os.makedirs(tmp_log_folder) - log_stream = open(log_path, 'w') - log("Logs are saved under {0}".format(log_path), level=3) - print("Install script run at {0} on {1}\n\n".format(execution_time, execution_day), file=log_stream) - - -def check_environment(): - needed_environ_vars = ['LANG', 'LC_ALL'] - message = '' - - for var in needed_environ_vars: - if var not in os.environ: - message += "\nexport {0}=C.UTF-8".format(var) - - if message: - log("Bench's CLI needs these to be defined!", level=3) - log("Run the following commands in shell: {0}".format(message), level=2) - sys.exit() - - -def check_system_package_managers(): - if 'Darwin' in os.uname(): - if not shutil.which('brew'): - raise Exception(''' - Please install brew package manager before proceeding with bench setup. Please run following - to install brew package manager on your machine, - - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - ''') - if 'Linux' in os.uname(): - if not any([shutil.which(x) for x in ['apt-get', 'yum']]): - raise Exception('Cannot find any compatible package manager!') - - -def check_distribution_compatibility(): - dist_name, dist_version = get_distribution_info() - supported_dists = { - 'macos': [10.9, 10.10, 10.11, 10.12], - 'ubuntu': [14, 15, 16, 18, 19, 20], - 'debian': [8, 9, 10], - 'centos': [7] - } - - log("Checking System Compatibility...") - if dist_name in supported_dists: - if float(dist_version) in supported_dists[dist_name]: - log("{0} {1} is compatible!".format(dist_name, dist_version), level=1) - else: - log("{0} {1} is detected".format(dist_name, dist_version), level=1) - log("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) - else: - log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) - - -def import_with_install(package): - # copied from https://discuss.erpnext.com/u/nikunj_patel - # https://discuss.erpnext.com/t/easy-install-setup-guide-for-erpnext-installation-on-ubuntu-20-04-lts-with-some-modification-of-course/62375/5 - # need to move to top said v13 for fully python3 era - import importlib - - try: - importlib.import_module(package) - except ImportError: - # caveat : pip3 must be installed - - import pip - - pip.main(['install', package]) - finally: - globals()[package] = importlib.import_module(package) - - -def get_distribution_info(): - # return distribution name and major version - if platform.system() == "Linux": - if distro_required: - current_dist = distro.linux_distribution(full_distribution_name=True) - else: - current_dist = platform.dist() - - return current_dist[0].lower(), current_dist[1].rsplit('.')[0] - - elif platform.system() == "Darwin": - current_dist = platform.mac_ver() - return "macos", current_dist[0].rsplit('.', 1)[0] - - -def run_os_command(command_map): - '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'}''' - success = True - - for executable, commands in command_map.items(): - if shutil.which(executable): - if isinstance(commands, str): - commands = [commands] - - for command in commands: - returncode = subprocess.check_call(command, shell=True, stdout=log_stream, stderr=sys.stderr) - success = success and (returncode == 0) - - return success - - -def install_prerequisites(): - # pre-requisites for bench repo cloning - run_os_command({ - 'apt-get': [ - 'sudo apt-get update', - 'sudo apt-get install -y git build-essential python3-setuptools python3-dev libffi-dev' - ], - 'yum': [ - 'sudo yum groupinstall -y "Development tools"', - 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' - ] - }) - - # until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source - if platform.machine() == 'aarch64': - log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...") - run_os_command({ - 'apt-get': ['sudo apt-get install -y libpq-dev libssl-dev'], - 'yum': ['sudo yum install -y libpq-devel openssl-devel'] - }) - - install_package('curl') - install_package('wget') - install_package('git') - install_package('pip3', 'python3-pip') - - run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade pip setuptools-rust" - }) - success = run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15" - }) - - if not (success or shutil.which('ansible')): - could_not_install('Ansible') - - -def could_not_install(package): - raise Exception('Could not install {0}. Please install it manually.'.format(package)) - - -def is_sudo_user(): - return os.geteuid() == 0 - - -def install_package(package, package_name=None): - if shutil.which(package): - log("{0} already installed!".format(package), level=1) - else: - log("Installing {0}...".format(package)) - package_name = package_name or package - success = run_os_command({ - 'apt-get': ['sudo apt-get install -y {0}'.format(package_name)], - 'yum': ['sudo yum install -y {0}'.format(package_name)], - 'brew': ['brew install {0}'.format(package_name)] - }) - if success: - log("{0} installed!".format(package), level=1) - return success - could_not_install(package) - - -def install_bench(args): - # clone bench repo - if not args.run_travis: - clone_bench_repo(args) - - if not args.user: - if args.production: - args.user = 'frappe' - - elif 'SUDO_USER' in os.environ: - args.user = os.environ['SUDO_USER'] - - else: - args.user = getpass.getuser() - - if args.user == 'root': - raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo or pass --user=USER') - - # Python executable - dist_name, dist_version = get_distribution_info() - if dist_name=='centos': - args.python = 'python3.6' - else: - args.python = 'python3' - - # create user if not exists - extra_vars = vars(args) - extra_vars.update(frappe_user=args.user) - - extra_vars.update(user_directory=get_user_home_directory(args.user)) - - if os.path.exists(tmp_bench_repo): - repo_path = tmp_bench_repo - else: - repo_path = os.path.join(os.path.expanduser('~'), 'bench') - - extra_vars.update(repo_path=repo_path) - run_playbook('create_user.yml', extra_vars=extra_vars) - - extra_vars.update(get_passwords(args)) - if args.production: - extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) - - if args.version <= 10: - frappe_branch = "{0}.x.x".format(args.version) - erpnext_branch = "{0}.x.x".format(args.version) - else: - frappe_branch = "version-{0}".format(args.version) - erpnext_branch = "version-{0}".format(args.version) - - # Allow override of frappe_branch and erpnext_branch, regardless of args.version (which always has a default set) - if args.frappe_branch: - frappe_branch = args.frappe_branch - if args.erpnext_branch: - erpnext_branch = args.erpnext_branch - - extra_vars.update(frappe_branch=frappe_branch) - extra_vars.update(erpnext_branch=erpnext_branch) - - bench_name = 'frappe-bench' if not args.bench_name else args.bench_name - extra_vars.update(bench_name=bench_name) - - # Will install ERPNext production setup by default - if args.without_erpnext: - log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext will not be installed due to --without-erpnext".format(bench_name=bench_name, frappe_branch=frappe_branch)) - else: - log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext Branch: {erpnext_branch}".format(bench_name=bench_name, frappe_branch=frappe_branch, erpnext_branch=erpnext_branch)) - run_playbook('site.yml', sudo=True, extra_vars=extra_vars) - - if os.path.exists(tmp_bench_repo): - shutil.rmtree(tmp_bench_repo) - - -def clone_bench_repo(args): - '''Clones the bench repository in the user folder''' - branch = args.bench_branch or 'develop' - repo_url = args.repo_url or 'https://github.com/frappe/bench' - - if os.path.exists(tmp_bench_repo): - log('Not cloning already existing Bench repository at {tmp_bench_repo}'.format(tmp_bench_repo=tmp_bench_repo)) - return 0 - elif args.without_bench_setup: - clone_path = os.path.join(os.path.expanduser('~'), 'bench') - log('--without-bench-setup specified, clone path is: {clone_path}'.format(clone_path=clone_path)) - else: - clone_path = tmp_bench_repo - # Not logging repo_url to avoid accidental credential leak in case credential is embedded in URL - log('Cloning bench repository branch {branch} into {clone_path}'.format(branch=branch, clone_path=clone_path)) - - success = run_os_command( - {'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( - repo_url=repo_url, bench_repo=clone_path, branch=branch)} - ) - - return success - - -def passwords_didnt_match(context=''): - log("{} passwords did not match!".format(context), level=3) - - -def get_passwords(args): - """ - Returns a dict of passwords for further use - and creates passwords.txt in the bench user's home directory - """ - log("Input MySQL and Frappe Administrator passwords:") - ignore_prompt = args.run_travis or args.without_bench_setup - mysql_root_password, admin_password = '', '' - passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt') - - if not ignore_prompt: - # set passwords from existing passwords.txt - if os.path.isfile(passwords_file_path): - with open(passwords_file_path, 'r') as f: - passwords = json.load(f) - mysql_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password'] - - # set passwords from cli args - if args.mysql_root_password: - mysql_root_password = args.mysql_root_password - if args.admin_password: - admin_password = args.admin_password - - # prompt for passwords - pass_set = True - while pass_set: - # mysql root password - if not mysql_root_password: - mysql_root_password = getpass.unix_getpass(prompt='Please enter mysql root password: ') - conf_mysql_passwd = getpass.unix_getpass(prompt='Re-enter mysql root password: ') - - if mysql_root_password != conf_mysql_passwd or mysql_root_password == '': - passwords_didnt_match("MySQL") - mysql_root_password = '' - continue - - # admin password, only needed if we're also creating a site - if not admin_password and not args.without_site: - admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ') - conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') - - if admin_password != conf_admin_passswd or admin_password == '': - passwords_didnt_match("Administrator") - admin_password = '' - continue - elif args.without_site: - log("Not creating a new site due to --without-site") - - pass_set = False - else: - mysql_root_password = admin_password = 'travis' - - passwords = { - 'mysql_root_password': mysql_root_password, - 'admin_password': admin_password - } - - if not ignore_prompt: - with open(passwords_file_path, 'w') as f: - json.dump(passwords, f, indent=1) - - log('Passwords saved at ~/passwords.txt') - - return passwords - - -def get_extra_vars_json(extra_args): - # We need to pass production as extra_vars to the playbook to execute conditionals in the - # playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON. - json_path = os.path.join('/', 'tmp', 'extra_vars.json') - extra_vars = dict(list(extra_args.items())) - - with open(json_path, mode='w') as j: - json.dump(extra_vars, j, indent=1, sort_keys=True) - - return ('@' + json_path) - -def get_user_home_directory(user): - # Return home directory /home/USERNAME or anything else defined as home directory in - # passwd for user. - return os.path.expanduser('~'+user) - - -def run_playbook(playbook_name, sudo=False, extra_vars=None): - args = ['ansible-playbook', '-c', 'local', playbook_name , '-vvvv'] - - if extra_vars: - args.extend(['-e', get_extra_vars_json(extra_vars)]) - - if sudo: - user = extra_vars.get('user') or getpass.getuser() - args.extend(['--become', '--become-user={0}'.format(user)]) - - if os.path.exists(tmp_bench_repo): - cwd = tmp_bench_repo - else: - cwd = os.path.join(os.path.expanduser('~'), 'bench') - - playbooks_locations = [os.path.join(cwd, 'bench', 'playbooks'), os.path.join(cwd, 'playbooks')] - playbooks_folder = [x for x in playbooks_locations if os.path.exists(x)][0] - - success = subprocess.check_call(args, cwd=playbooks_folder, stdout=log_stream, stderr=sys.stderr) - return success - - -def setup_script_requirements(): - if distro_required: - install_package('pip3', 'python3-pip') - import_with_install('distro') - - -def parse_commandline_args(): - import argparse - - parser = argparse.ArgumentParser(description='Frappe Installer') - # Arguments develop and production are mutually exclusive both can't be specified together. - # Hence, we need to create a group for discouraging use of both options at the same time. - args_group = parser.add_mutually_exclusive_group() - - args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup') - args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench') - parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specify name for your first ERPNext site') - parser.add_argument('--without-site', dest='without_site', action='store_true', default=False, help='Do not create a new site') - parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode') - parser.add_argument('--user', dest='user', help='Install frappe-bench for this user') - parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') - parser.add_argument('--repo-url', dest='repo_url', help='Clone bench from the given url') - parser.add_argument('--frappe-repo-url', dest='frappe_repo_url', action='store', default='https://github.com/frappe/frappe', help='Clone frappe from the given url') - parser.add_argument('--frappe-branch', dest='frappe_branch', action='store', help='Clone a particular branch of frappe') - parser.add_argument('--erpnext-repo-url', dest='erpnext_repo_url', action='store', default='https://github.com/frappe/erpnext', help='Clone erpnext from the given url') - parser.add_argument('--erpnext-branch', dest='erpnext_branch', action='store', help='Clone a particular branch of erpnext') - parser.add_argument('--without-erpnext', dest='without_erpnext', action='store_true', default=False, help='Prevent fetching ERPNext') - # direct provision to install versions - parser.add_argument('--version', dest='version', action='store', default=13, type=int, help='Clone particular version of frappe and erpnext') - # To enable testing of script using Travis, this should skip the prompt - parser.add_argument('--run-travis', dest='run_travis', action='store_true', default=False, help=argparse.SUPPRESS) - parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False, help=argparse.SUPPRESS) - # whether to overwrite an existing bench - parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='Whether to overwrite an existing bench') - # set passwords - parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') - parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.4', help='Specify mariadb version') - parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') - parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench') - # Python interpreter to be used - parser.add_argument('--python', dest='python', default='python3', help=argparse.SUPPRESS) - # LXC Support - parser.add_argument('--container', dest='container', default=False, action='store_true', help='Use if you\'re creating inside LXC') - - args = parser.parse_args() - - return args - - -if __name__ == '__main__': - if sys.version[0] == '2': - if not os.environ.get('CI'): - if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": - sys.exit() - - try: - from distutils.spawn import find_executable - except ImportError: - try: - subprocess.check_call('pip install --upgrade setuptools') - except subprocess.CalledProcessError: - print("Install distutils or use Python3 to run the script") - sys.exit(1) - - shutil.which = find_executable - - if not is_sudo_user(): - log("Please run this script as a non-root user with sudo privileges", level=3) - sys.exit() - - args = parse_commandline_args() - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - setup_log_stream(args) - install_prerequisites() - setup_script_requirements() - check_distribution_compatibility() - check_system_package_managers() - check_environment() - install_bench(args) - - log("Bench + Frappe + ERPNext has been successfully installed!") From c4305fd528f30765e2b8a59c4979ff286433482c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 23 Nov 2022 14:35:01 +0530 Subject: [PATCH 05/54] style: format everything w black --- bench/config/common_site_config.py | 4 +++- bench/config/supervisor.py | 15 ++++++++++++--- bench/config/systemd.py | 15 ++++++++++++--- bench/utils/__init__.py | 5 ++++- bench/utils/app.py | 4 +++- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 41242aad3..df58daf13 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -17,6 +17,7 @@ DEFAULT_MAX_REQUESTS = 5000 + def setup_config(bench_path): make_pid_folder(bench_path) bench_config = get_config(bench_path) @@ -62,9 +63,11 @@ def get_gunicorn_workers(): return {"gunicorn_workers": multiprocessing.cpu_count() * 2 + 1} + def compute_max_requests_jitter(max_requests: int) -> int: return int(max_requests * 0.1) + def get_default_max_requests(worker_count: int): """Get max requests and jitter config based on number of available workers.""" @@ -75,7 +78,6 @@ def get_default_max_requests(worker_count: int): return DEFAULT_MAX_REQUESTS - def update_config_for_frappe(config, bench_path): ports = make_ports(bench_path) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 58015b81e..b78c04702 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,7 +8,12 @@ from bench.app import use_rq from bench.utils import get_bench_name, which from bench.bench import Bench -from bench.config.common_site_config import update_config, get_gunicorn_workers, get_default_max_requests, compute_max_requests_jitter +from bench.config.common_site_config import ( + update_config, + get_gunicorn_workers, + get_default_max_requests, + compute_max_requests_jitter, +) # imports - third party imports import click @@ -26,8 +31,12 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals template = bench.config.env().get_template("supervisor.conf") bench_dir = os.path.abspath(bench_path) - web_worker_count = config.get("gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"]) - max_requests = config.get("gunicorn_max_requests", get_default_max_requests(web_worker_count)) + web_worker_count = config.get( + "gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"] + ) + max_requests = config.get( + "gunicorn_max_requests", get_default_max_requests(web_worker_count) + ) config = template.render( **{ diff --git a/bench/config/systemd.py b/bench/config/systemd.py index a677391cd..0d4e1243c 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -9,7 +9,12 @@ import bench from bench.app import use_rq from bench.bench import Bench -from bench.config.common_site_config import get_gunicorn_workers, update_config, get_default_max_requests, compute_max_requests_jitter +from bench.config.common_site_config import ( + get_gunicorn_workers, + update_config, + get_default_max_requests, + compute_max_requests_jitter, +) from bench.utils import exec_cmd, which, get_bench_name @@ -61,8 +66,12 @@ def generate_systemd_config( get_bench_name(bench_path) + "-frappe-long-worker@" + str(i + 1) + ".service" ) - web_worker_count = config.get("gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"]) - max_requests = config.get("gunicorn_max_requests", get_default_max_requests(web_worker_count)) + web_worker_count = config.get( + "gunicorn_workers", get_gunicorn_workers()["gunicorn_workers"] + ) + max_requests = config.get( + "gunicorn_max_requests", get_default_max_requests(web_worker_count) + ) bench_info = { "bench_dir": bench_dir, diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index ff8f8fcdb..b61f686e1 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -125,7 +125,10 @@ def check_latest_version(): local_version = Version(VERSION) if pypi_version > local_version: - log(f"A newer version of bench is available: {local_version} → {pypi_version}", stderr=True) + log( + f"A newer version of bench is available: {local_version} → {pypi_version}", + stderr=True, + ) def pause_exec(seconds=10): diff --git a/bench/utils/app.py b/bench/utils/app.py index 790792dab..5541b548e 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -185,7 +185,9 @@ def get_required_deps(org, name, branch, deps="hooks.py"): res = requests.get(url=git_api_url, params=params).json() if "message" in res: - git_url = f"https://raw.githubusercontent.com/{org}/{name}/{params['ref']}/{name}/{deps}" + git_url = ( + f"https://raw.githubusercontent.com/{org}/{name}/{params['ref']}/{name}/{deps}" + ) return requests.get(git_url).text return base64.decodebytes(res["content"].encode()).decode() From 9c80f5d24f3e9a88531c7300f95ee7dc3839256d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 24 Nov 2022 15:29:34 +0530 Subject: [PATCH 06/54] perf: single worker in development install (#1392) Most developers don't need 3 separate workers in development. This changes procfile to use single worker to consume from all queues in development. Pros: - Lighter development setups Cons: - Not "equivalent to production" - not required in most cases so eh. You can still edit procfile to start whatever process you want anyway. --- bench/config/templates/Procfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index d9391087c..899d1290c 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -11,9 +11,7 @@ watch: bench watch {% endif %} {% if use_rq -%} schedule: bench schedule -worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log -worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log -worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log +worker: bench worker 1>> logs/worker.log 2>> logs/worker.error.log {% for worker_name, worker_details in workers.items() %} worker_{{ worker_name }}: bench worker --queue {{ worker_name }} 1>> logs/worker.log 2>> logs/worker.error.log {% endfor %} From c59d1edee5e2c22f55f91ac946df8e09535c8927 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 28 Nov 2022 13:05:00 +0530 Subject: [PATCH 07/54] fix: restart proc manager if set in config (#1391) --- bench/bench.py | 2 +- bench/config/systemd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 8586c7124..e799838a3 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -147,7 +147,7 @@ def reload(self, web=False, supervisor=True, systemd=True): if conf.get("developer_mode"): restart_process_manager(bench_path=self.name, web_workers=web) - if supervisor and conf.get("restart_supervisor_on_update"): + if supervisor or conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=self.name, web_workers=web) if systemd and conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=self.name, web_workers=web) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 0d4e1243c..a19de6620 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -105,7 +105,7 @@ def generate_systemd_config( setup_web_config(bench_info, bench_path) setup_redis_config(bench_info, bench_path) - update_config({"restart_systemd_on_update": True}, bench_path=bench_path) + update_config({"restart_systemd_on_update": False}, bench_path=bench_path) update_config({"restart_supervisor_on_update": False}, bench_path=bench_path) From 7f1c5ad9c673f158885b083c39000d071f6400a0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 29 Nov 2022 12:23:13 +0530 Subject: [PATCH 08/54] fix: migrate_env silent failures --- bench/utils/bench.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 5467256a7..eb2b00eba 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -223,7 +223,8 @@ def migrate_env(python, backup=False): logger.log(f"Migration Successful to {python}") except Exception: if venv_creation or packages_setup: - logger.warning("Migration Error") + logger.warning("Migration Error", exc_info=True) + raise def validate_upgrade(from_ver, to_ver, bench_path="."): From 61e0f4acbf4b7e95da2d1d0b26516cd41bfc6eed Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 29 Nov 2022 12:23:49 +0530 Subject: [PATCH 09/54] fix: correct python path for migrate_env Currently it's executing env directory which would never work. --- bench/utils/bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index eb2b00eba..742dc5baf 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -218,7 +218,7 @@ def migrate_env(python, backup=False): venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}") apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f"{pvenv} -m pip install --upgrade {apps}") + packages_setup = exec_cmd(f"{pvenv}/bin/python -m pip install --upgrade {apps}") logger.log(f"Migration Successful to {python}") except Exception: From 4f0193ca9349d971e1ce3087c556b9710bc5db69 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 29 Nov 2022 12:33:53 +0530 Subject: [PATCH 10/54] fix: install one app at a time, frappe first --- bench/utils/bench.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 742dc5baf..8922814e2 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -212,18 +212,23 @@ def migrate_env(python, backup=False): shutil.move(dest, target) # Create virtualenv using specified python - venv_creation, packages_setup = 1, 1 + def _install_app(app): + app_path = f"-e {os.path.join('apps', app)}" + exec_cmd(f"{pvenv}/bin/python -m pip install --upgrade {app_path}") + try: logger.log(f"Setting up a New Virtual {python} Environment") - venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}") + exec_cmd(f"{virtualenv} --python {python} {pvenv}") - apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f"{pvenv}/bin/python -m pip install --upgrade {apps}") + # Install frappe first + _install_app("frappe") + for app in bench.apps: + if str(app) != "frappe": + _install_app(app) logger.log(f"Migration Successful to {python}") except Exception: - if venv_creation or packages_setup: - logger.warning("Migration Error", exc_info=True) + logger.warning("Python env migration Error", exc_info=True) raise From e37df969c1641eda06ecf84f4ad63310f37f9183 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 29 Nov 2022 12:40:11 +0530 Subject: [PATCH 11/54] refactor(migrate_env): virtualenv -> venv --- bench/utils/bench.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 8922814e2..c324c063b 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -175,10 +175,6 @@ def migrate_env(python, backup=False): nvenv = "env" path = os.getcwd() python = which(python) - virtualenv = which("virtualenv") - if not virtualenv: - raise FileNotFoundError("`virtualenv` not found. Install it and try again.") - pvenv = os.path.join(path, nvenv) # Clear Cache before Bench Dies. @@ -218,7 +214,7 @@ def _install_app(app): try: logger.log(f"Setting up a New Virtual {python} Environment") - exec_cmd(f"{virtualenv} --python {python} {pvenv}") + exec_cmd(f"{python} -m venv {pvenv}") # Install frappe first _install_app("frappe") From 0e59159e1167357f6d8411708a93ad37cc4b959e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 29 Nov 2022 12:49:19 +0530 Subject: [PATCH 12/54] fix: dont attempt migrating to active virtualenv --- bench/utils/bench.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index c324c063b..62f3e5967 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -177,6 +177,15 @@ def migrate_env(python, backup=False): python = which(python) pvenv = os.path.join(path, nvenv) + if python.startswith(pvenv): + # The supplied python version is in active virtualenv which we are about to nuke. + click.secho( + "Python version supplied is present in currently sourced virtual environment.\n" + "`deactiviate` the current virtual environment before migrating environments.", + fg="yellow", + ) + sys.exit(1) + # Clear Cache before Bench Dies. try: config = bench.conf From 044e01befde722c20eb6adeb6a6f01b353291da1 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Thu, 1 Dec 2022 16:24:47 +0530 Subject: [PATCH 13/54] feat: New easy-install.py Supports docker container setup with production and dev instances --- easy-install.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 easy-install.py diff --git a/easy-install.py b/easy-install.py new file mode 100644 index 000000000..8a92a9d81 --- /dev/null +++ b/easy-install.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import os +import sys +import time +from shutil import which +import platform +from secrets import token_bytes +from base64 import b64encode + +CRED = "\033[31m" +CEND = "\033[0m" +CGRN = "\33[92m" +CYLW = "\33[93m" + + +def clone_frappe_docker_repo() -> None: + try: + subprocess.run( + ["git", "clone", "https://github.com/frappe/frappe_docker"], check=True + ) + except Exception as e: + print(f"\n{CRED}Cloning frappe_docker Failed{CEND}\n\n", e) + + +def write_to_env(wd, site,db_pass,email): + site_name = site if site != "" else "" + with open(os.path.join(wd, ".env"), "w") as f: + f.writelines( + [ + "FRAPPE_VERSION=v14.17.1\n", + "ERPNEXT_VERSION=v14.9.0\n", + f"DB_PASSWORD={db_pass}\n", + "DB_HOST=db\n", + "DB_PORT=3306\n", + "REDIS_CACHE=redis-cache:6379\n", + "REDIS_QUEUE=redis-queue:6379\n", + "REDIS_SOCKETIO=redis-socketio:6379\n", + f"LETSENCRYPT_EMAIL={email}\n" + f"FRAPPE_SITE_NAME_HEADER={site_name}", + ] + ) + +def generate_pass(length:int=9) -> str: + return b64encode(token_bytes(length)).decode() + + +def check_repo_exists() -> bool: + return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) + + +def setup_prod(project: str, sitename: str,email:str): + if check_repo_exists(): + compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") + docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") + admin_pass = generate_pass() + db_pass = generate_pass(6) + print( + f"\n{CYLW}Please refer to .example.env file in the frappe_docker folder to know which keys to set{CEND}\n\n" + ) + with open(compose_file_name, "w") as f: + if not os.path.exists(os.path.join(docker_repo_path, "/.env")): + write_to_env(docker_repo_path, sitename,db_pass,email) + print( + f"\n{CYLW}A .env file is generated with basic configs. Please edit it to fit your needs {CEND}\n" + ) + try: + # TODO: Include flags for non-https and non-erpnext installation + subprocess.run( + [ + which("docker"), + "compose", + "--project-name", + project, + "-f", + "compose.yaml", + "-f", + "overrides/compose.mariadb.yaml", + "-f", + "overrides/compose.redis.yaml", + # "-f", "overrides/compose.noproxy.yaml", TODO: Add support for local proxying without HTTPs + "-f", + "overrides/compose.erpnext.yaml", + "-f", + "overrides/compose.https.yaml", + "--env-file", + ".env", + "config", + ], + cwd=docker_repo_path, + stdout=f, + check=True, + ) + + except Exception as e: + print(f"\n{CRED}Generating Compose File failed{CEND}\n") + try: + subprocess.run( + [ + which("docker"), + "compose", + "-p", + project, + "-f", + compose_file_name, + "up", + "-d", + ], + check=True, + ) + + except Exception as e: + print( + f"{CRED} Docker Compose failed, please check the container logs{CEND}\n", + e, + ) + print(f"\n{CGRN}Creating site: {sitename} {CEND}\n") + with open(os.path.join(os.path.expanduser("~"),"passwords.txt"),"w") as f: + f.writelines(f"Administrator:{admin_pass}\n") + f.writelines(f"MariaDB Root Password:{db_pass}\n") + try: + subprocess.run([ + which("docker"),"compose", + "-p",project,"exec","backend", + "bench","new-site",sitename, + "--db-root-password",db_pass, + "--admin-password",admin_pass, + "--install-app","erpnext" + ],check=True) + except Exception as e: + print( + f"{CRED} Bench Site creation failed{CEND}\n", + e, + ) + else: + install_docker() + clone_frappe_docker_repo() + setup_prod(project, sitename,email) # Recursive + + +def setup_dev_instance(project: str): + if check_repo_exists(): + try: + subprocess.run( + [ + "docker", + "compose", + "-f", + "devcontainer-example/docker-compose.yml", + "--project-name", + project, + "up", + "-d", + ], + cwd=os.path.join(os.getcwd(), "frappe_docker"), + check=True, + ) + print( + f"{CYLW}Please go through the Development Documentation: https://github.com/frappe/frappe_docker/tree/main/development to fully complete the setup.{CEND}" + ) + except Exception as e: + print(f"{CRED}Setting Up Development Environment Failed\n{CEND}", e) + else: + install_docker() + clone_frappe_docker_repo() + setup_dev_instance(project) # Recursion on goes brrrr + + +def install_docker(): + if which("docker") is None: + print(f"{CGRN}Docker is not installed, Installing Docker...{CEND}") + if platform.system() == "Darwin" or platform.system() == "Windows": + print( + f"""{CRED} + This script doesn't install Docker on {"Mac" if platform.system()=="Darwin" else "Windows"}. + + Please go through the Docker Installation docs for your system and run this script again{CEND}""" + ) + exit(1) + try: + ps = subprocess.run( + ["curl", "-fsSL", "https://get.docker.com"], + capture_output=True, + check=True, + ) + subprocess.run(["/bin/bash"], input=ps.stdout, capture_output=True) + subprocess.run( + ["sudo", "usermod", "-aG", "docker", str(os.getenv("USER"))], check=True + ) + print(f"{CYLW}Waiting Docker to start{CEND}") + time.sleep(10) + subprocess.run( + ["sudo", "systemctl", "restart", "docker.service"], check=True + ) + except Exception as e: + print(f"{CRED}Failed to Install Docker{CYLW}\n", e) + print( + f"\n\n {CYLW}Try Installing Docker Manually and re-run this script again{CEND}\n\n" + ) + exit(1) + else: + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Install Frappe with Docker") + parser.add_argument( + "-p", "--prod", help="Setup Production System", action="store_true" + ) + parser.add_argument( + "-d", "--dev", help="Setup Development System", action="store_true" + ) + parser.add_argument( + "-s", + "--sitename", + help="The Site Name for your production site", + default="site1.local", + ) + parser.add_argument( + "-n", + "--project", + help="Project Name", + default="frappe", + ) + parser.add_argument( + "--email", + help="Add email for the SSL.", + required="--prod" in sys.argv + ) + args = parser.parse_args() + if args.dev: + print(f"\n{CGRN}Setting Up Development Instance{CEND}\n") + setup_dev_instance(args.project) + elif args.prod: + print(f"\n{CGRN}Setting Up Production Instance{CEND}\n") + if "example.com" in args.email: + print(f"{CRED} Emails with example.com not acceptable{CEND}") + setup_prod(args.project, args.sitename, args.email) \ No newline at end of file From 7c8ce31a8da880f0d8f60818d98adc3ce09ca0e2 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Mon, 5 Dec 2022 00:37:39 +0530 Subject: [PATCH 14/54] chore: Added logging and removed git dependency Also added - New methods for printing - Better env file management - Better password generation - Added zip repository downloads - default container version set to v14 --- easy-install.py | 252 +++++++++++++++++++++++++++++------------------- 1 file changed, 153 insertions(+), 99 deletions(-) diff --git a/easy-install.py b/easy-install.py index 8a92a9d81..9911ee6a5 100644 --- a/easy-install.py +++ b/easy-install.py @@ -4,67 +4,113 @@ import os import sys import time -from shutil import which +import urllib.request +from shutil import which, unpack_archive, move import platform -from secrets import token_bytes -from base64 import b64encode +from hashlib import sha224 +import logging -CRED = "\033[31m" -CEND = "\033[0m" -CGRN = "\33[92m" -CYLW = "\33[93m" +logging.basicConfig( + filename="easy-install.log", + filemode="w", + format="%(levelname)s - %(message)s", + level=logging.INFO, +) + + +def cprint(*args, level: int = 1): + """ + logs colorful messages + level = 1 : RED + level = 2 : GREEN + level = 3 : YELLOW + + default level = 1 + """ + CRED = "\033[31m" + CGRN = "\33[92m" + CYLW = "\33[93m" + reset = "\033[0m" + message = " ".join(map(str, args)) + if level == 1: + print(CRED, message, reset) + if level == 2: + print(CGRN, message, reset) + if level == 3: + print(CYLW, message, reset) def clone_frappe_docker_repo() -> None: try: - subprocess.run( - ["git", "clone", "https://github.com/frappe/frappe_docker"], check=True + urllib.request.urlretrieve( + "https://github.com/frappe/frappe_docker/archive/refs/heads/main.zip", + "frappe_docker.zip", ) + logging.info("Downloaded frappe_docker zip file from GitHub") + unpack_archive( + "frappe_docker.zip", "." + ) # Unzipping the frappe_docker.zip creates a folder "frappe_docker-main" + move("frappe_docker-main", "frappe_docker") + logging.info("Unzipped and Renamed frappe_docker") + os.remove("frappe_docker.zip") + logging.info("Removed the downloaded zip file") except Exception as e: - print(f"\n{CRED}Cloning frappe_docker Failed{CEND}\n\n", e) + logging.error("Download and unzip failed", exc_info=True) + cprint("\nCloning frappe_docker Failed\n\n", "[ERROR]: ", e, level=1) -def write_to_env(wd, site,db_pass,email): - site_name = site if site != "" else "" +def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) -> None: + site_name = site or "" with open(os.path.join(wd, ".env"), "w") as f: f.writelines( [ - "FRAPPE_VERSION=v14.17.1\n", - "ERPNEXT_VERSION=v14.9.0\n", + "FRAPPE_VERSION=v14\n", # Defaults to latest version of Frappe + "ERPNEXT_VERSION=v14\n", # defaults to latest version of ERPNext f"DB_PASSWORD={db_pass}\n", "DB_HOST=db\n", "DB_PORT=3306\n", "REDIS_CACHE=redis-cache:6379\n", "REDIS_QUEUE=redis-queue:6379\n", "REDIS_SOCKETIO=redis-socketio:6379\n", - f"LETSENCRYPT_EMAIL={email}\n" - f"FRAPPE_SITE_NAME_HEADER={site_name}", + f"LETSENCRYPT_EMAIL={email}\n", + f"FRAPPE_SITE_NAME_HEADER={site_name}\n", + f"SITE_ADMIN_PASS={admin_pass}", ] ) -def generate_pass(length:int=9) -> str: - return b64encode(token_bytes(length)).decode() - + +def generate_pass(length: int = 12) -> str: + return sha224(repr(time.time()).encode()).hexdigest()[:length] + def check_repo_exists() -> bool: return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) -def setup_prod(project: str, sitename: str,email:str): +def setup_prod(project: str, sitename: str, email: str) -> None: if check_repo_exists(): - compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") + compose_file_name = os.path.join( + os.path.expanduser("~"), f"{project}-compose.yml" + ) docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") - admin_pass = generate_pass() - db_pass = generate_pass(6) - print( - f"\n{CYLW}Please refer to .example.env file in the frappe_docker folder to know which keys to set{CEND}\n\n" + cprint( + "\nPlease refer to .example.env file in the frappe_docker folder to know which keys to set\n\n", + level=3, ) with open(compose_file_name, "w") as f: - if not os.path.exists(os.path.join(docker_repo_path, "/.env")): - write_to_env(docker_repo_path, sitename,db_pass,email) - print( - f"\n{CYLW}A .env file is generated with basic configs. Please edit it to fit your needs {CEND}\n" + admin_pass = generate_pass() + db_pass = generate_pass(9) + if not os.path.exists(os.path.join(docker_repo_path, ".env")): + write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) + cprint( + "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", + level=3, ) + with open( + os.path.join(os.path.expanduser("~"), "passwords.txt"), "w" + ) as en: + en.writelines(f"Administrator:{admin_pass}\n") + en.writelines(f"MariaDB Root Password:{db_pass}\n") try: # TODO: Include flags for non-https and non-erpnext installation subprocess.run( @@ -94,7 +140,9 @@ def setup_prod(project: str, sitename: str,email:str): ) except Exception as e: - print(f"\n{CRED}Generating Compose File failed{CEND}\n") + logging.error("Docker Compose generation failed", exc_info=True) + cprint("\nGenerating Compose File failed\n") + sys.exit(1) try: subprocess.run( [ @@ -109,34 +157,43 @@ def setup_prod(project: str, sitename: str,email:str): ], check=True, ) - + logging.info(f"Docker Compose file generated at ~/{project}-compose.yml") + except Exception as e: - print( - f"{CRED} Docker Compose failed, please check the container logs{CEND}\n", - e, - ) - print(f"\n{CGRN}Creating site: {sitename} {CEND}\n") - with open(os.path.join(os.path.expanduser("~"),"passwords.txt"),"w") as f: - f.writelines(f"Administrator:{admin_pass}\n") - f.writelines(f"MariaDB Root Password:{db_pass}\n") + logging.error("Prod docker-compose failed", exc_info=True) + cprint(" Docker Compose failed, please check the container logs\n", e) + + cprint(f"\nCreating site: {sitename} \n", level=3) + try: - subprocess.run([ - which("docker"),"compose", - "-p",project,"exec","backend", - "bench","new-site",sitename, - "--db-root-password",db_pass, - "--admin-password",admin_pass, - "--install-app","erpnext" - ],check=True) - except Exception as e: - print( - f"{CRED} Bench Site creation failed{CEND}\n", - e, + subprocess.run( + [ + which("docker"), + "compose", + "-p", + project, + "exec", + "backend", + "bench", + "new-site", + sitename, + "--db-root-password", + db_pass, + "--admin-password", + admin_pass, + "--install-app", + "erpnext", + ], + check=True, ) + logging.info("New site creation completed") + except Exception as e: + logging.error("Bench site creation failed", exc_info=True) + cprint("Bench Site creation failed\n", e) else: install_docker() clone_frappe_docker_repo() - setup_prod(project, sitename,email) # Recursive + setup_prod(project, sitename, email) # Recursive def setup_dev_instance(project: str): @@ -156,11 +213,14 @@ def setup_dev_instance(project: str): cwd=os.path.join(os.getcwd(), "frappe_docker"), check=True, ) - print( - f"{CYLW}Please go through the Development Documentation: https://github.com/frappe/frappe_docker/tree/main/development to fully complete the setup.{CEND}" + cprint( + "Please go through the Development Documentation: https://github.com/frappe/frappe_docker/tree/main/development to fully complete the setup.", + level=2, ) + logging.info("Development Setup completed") except Exception as e: - print(f"{CRED}Setting Up Development Environment Failed\n{CEND}", e) + logging.error("Dev Environment setup failed", exc_info=True) + cprint("Setting Up Development Environment Failed\n", e) else: install_docker() clone_frappe_docker_repo() @@ -168,39 +228,37 @@ def setup_dev_instance(project: str): def install_docker(): - if which("docker") is None: - print(f"{CGRN}Docker is not installed, Installing Docker...{CEND}") - if platform.system() == "Darwin" or platform.system() == "Windows": - print( - f"""{CRED} - This script doesn't install Docker on {"Mac" if platform.system()=="Darwin" else "Windows"}. - - Please go through the Docker Installation docs for your system and run this script again{CEND}""" - ) - exit(1) - try: - ps = subprocess.run( - ["curl", "-fsSL", "https://get.docker.com"], - capture_output=True, - check=True, - ) - subprocess.run(["/bin/bash"], input=ps.stdout, capture_output=True) - subprocess.run( - ["sudo", "usermod", "-aG", "docker", str(os.getenv("USER"))], check=True - ) - print(f"{CYLW}Waiting Docker to start{CEND}") - time.sleep(10) - subprocess.run( - ["sudo", "systemctl", "restart", "docker.service"], check=True - ) - except Exception as e: - print(f"{CRED}Failed to Install Docker{CYLW}\n", e) - print( - f"\n\n {CYLW}Try Installing Docker Manually and re-run this script again{CEND}\n\n" - ) - exit(1) - else: + if which("docker") is not None: return + cprint("Docker is not installed, Installing Docker...", level=3) + logging.info("Docker not found, installing Docker") + if platform.system() == "Darwin" or platform.system() == "Windows": + print( + f""" + This script doesn't install Docker on {"Mac" if platform.system()=="Darwin" else "Windows"}. + + Please go through the Docker Installation docs for your system and run this script again""" + ) + logging.debug("Docker setup failed due to platform is not Linux") + sys.exit(1) + try: + ps = subprocess.run( + ["curl", "-fsSL", "https://get.docker.com"], + capture_output=True, + check=True, + ) + subprocess.run(["/bin/bash"], input=ps.stdout, capture_output=True) + subprocess.run( + ["sudo", "usermod", "-aG", "docker", str(os.getenv("USER"))], check=True + ) + cprint("Waiting Docker to start", level=3) + time.sleep(10) + subprocess.run(["sudo", "systemctl", "restart", "docker.service"], check=True) + except Exception as e: + logging.error("Installing Docker failed", exc_info=True) + cprint("Failed to Install Docker\n", e) + cprint("\n Try Installing Docker Manually and re-run this script again\n") + sys.exit(1) if __name__ == "__main__": @@ -217,23 +275,19 @@ def install_docker(): help="The Site Name for your production site", default="site1.local", ) + parser.add_argument("-n", "--project", help="Project Name", default="frappe") parser.add_argument( - "-n", - "--project", - help="Project Name", - default="frappe", - ) - parser.add_argument( - "--email", - help="Add email for the SSL.", - required="--prod" in sys.argv + "--email", help="Add email for the SSL.", required="--prod" in sys.argv ) args = parser.parse_args() if args.dev: - print(f"\n{CGRN}Setting Up Development Instance{CEND}\n") + cprint("\nSetting Up Development Instance\n", level=2) + logging.info("Running Development Setup") setup_dev_instance(args.project) elif args.prod: - print(f"\n{CGRN}Setting Up Production Instance{CEND}\n") + cprint("\nSetting Up Production Instance\n", level=2) + logging.info("Running Production Setup") if "example.com" in args.email: - print(f"{CRED} Emails with example.com not acceptable{CEND}") + cprint("Emails with example.com not acceptable", level=1) + sys.exit(1) setup_prod(args.project, args.sitename, args.email) \ No newline at end of file From 3ff7bfb35f17bd8d8c3f50e16c41280449f63569 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Tue, 6 Dec 2022 19:33:47 +0530 Subject: [PATCH 15/54] chore: get exact versions from example.env file --- easy-install.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/easy-install.py b/easy-install.py index 9911ee6a5..fb7424625 100644 --- a/easy-install.py +++ b/easy-install.py @@ -5,10 +5,11 @@ import sys import time import urllib.request -from shutil import which, unpack_archive, move +import logging import platform +from shutil import which, unpack_archive, move from hashlib import sha224 -import logging +from typing import Dict logging.basicConfig( filename="easy-install.log", @@ -58,14 +59,24 @@ def clone_frappe_docker_repo() -> None: logging.error("Download and unzip failed", exc_info=True) cprint("\nCloning frappe_docker Failed\n\n", "[ERROR]: ", e, level=1) +def get_latest_version(dir) -> Dict: + env_vars ={} + with open(os.path.join(dir,"example.env")) as f: + for line in f: + if line.startswith('#') or not line.strip(): + continue + key, value = line.strip().split('=', 1) + env_vars[key] = value + return env_vars def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) -> None: site_name = site or "" + example_env = get_latest_version(wd) with open(os.path.join(wd, ".env"), "w") as f: f.writelines( [ - "FRAPPE_VERSION=v14\n", # Defaults to latest version of Frappe - "ERPNEXT_VERSION=v14\n", # defaults to latest version of ERPNext + f"FRAPPE_VERSION={example_env['FRAPPE_VERSION']}\n", # Defaults to latest version of Frappe + f"ERPNEXT_VERSION={example_env['ERPNEXT_VERSION']}\n", # defaults to latest version of ERPNext f"DB_PASSWORD={db_pass}\n", "DB_HOST=db\n", "DB_PORT=3306\n", @@ -98,9 +109,10 @@ def setup_prod(project: str, sitename: str, email: str) -> None: level=3, ) with open(compose_file_name, "w") as f: - admin_pass = generate_pass() - db_pass = generate_pass(9) + # Writing to compose file if not os.path.exists(os.path.join(docker_repo_path, ".env")): + admin_pass = generate_pass() + db_pass = generate_pass(9) write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) cprint( "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", @@ -144,6 +156,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: cprint("\nGenerating Compose File failed\n") sys.exit(1) try: + # Starting with generated compose file subprocess.run( [ which("docker"), From 4996afb012ef1f05e56e576bb472df9217b00a6b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 12 Dec 2022 13:12:18 +0530 Subject: [PATCH 16/54] fix: install wheel in venv closes https://github.com/frappe/frappe/issues/18718 --- bench/bench.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bench/bench.py b/bench/bench.py index e799838a3..f58ddfd7a 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -361,6 +361,7 @@ def env(self, python="python3"): self.run(f"{venv} env", cwd=self.bench.name) self.pip() + self.wheel() if os.path.exists(frappe): self.run( @@ -398,6 +399,19 @@ def pip(self, verbose=False): f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip", cwd=self.bench.name ) + @step(title="Installing wheel", success="Installed wheel") + def wheel(self, verbose=False): + """Wheel is required for building old setup.py packages. + ref: https://github.com/pypa/pip/issues/8559""" + import bench.cli + + verbose = bench.cli.verbose or verbose + quiet_flag = "" if verbose else "--quiet" + + return self.run( + f"{self.bench.python} -m pip install {quiet_flag} wheel", cwd=self.bench.name + ) + def logging(self): from bench.utils import setup_logging From 7a63f4f461e7c62ec1973dde92620c27259f0d8c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 12 Dec 2022 14:08:04 +0530 Subject: [PATCH 17/54] fix: ignore supervisor restart failures where possible (#1400) * fix: ignore supervisor restart failures where possible Ignores proc manager restart failure during: - App install - App uninstall Reason: - You might not have setup prod yet - This is useful during docker image building where proc manager wont be running **yet**. * ci: dont fail-fast matrix has no dependency. --- .github/workflows/ci.yml | 2 ++ bench/app.py | 4 +++- bench/bench.py | 12 ++++++------ bench/utils/bench.py | 16 ++++++++++------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1040df3a0..ed5e8ebf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ jobs: timeout-minutes: 60 strategy: + fail-fast: false matrix: python-version: [ '3.7', '3.8', '3.9', '3.10' ] @@ -89,6 +90,7 @@ jobs: timeout-minutes: 60 strategy: + fail-fast: false matrix: python-version: [ '3.7', '3.10' ] diff --git a/bench/app.py b/bench/app.py index 07eb4799e..72f4825af 100755 --- a/bench/app.py +++ b/bench/app.py @@ -586,7 +586,9 @@ def install_app( build_assets(bench_path=bench_path, app=app) if restart_bench: - bench.reload() + # Avoiding exceptions here as production might not be set-up + # OR we might just be generating docker images. + bench.reload(_raise=False) def pull_apps(apps=None, bench_path=".", reset=False): diff --git a/bench/bench.py b/bench/bench.py index f58ddfd7a..cdba90180 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -44,8 +44,8 @@ class Base: - def run(self, cmd, cwd=None): - return exec_cmd(cmd, cwd=cwd or self.cwd) + def run(self, cmd, cwd=None, _raise=True): + return exec_cmd(cmd, cwd=cwd or self.cwd, _raise=_raise) class Validator: @@ -133,7 +133,7 @@ def uninstall(self, app, no_backup=False, force=False): raise self.apps.sync() # self.build() - removed because it seems unnecessary - self.reload() + self.reload(_raise=False) @step(title="Building Bench Assets", success="Bench Assets Built") def build(self): @@ -141,16 +141,16 @@ def build(self): run_frappe_cmd("build", bench_path=self.name) @step(title="Reloading Bench Processes", success="Bench Processes Reloaded") - def reload(self, web=False, supervisor=True, systemd=True): + def reload(self, web=False, supervisor=True, systemd=True, _raise=True): """If web is True, only web workers are restarted""" conf = self.conf if conf.get("developer_mode"): restart_process_manager(bench_path=self.name, web_workers=web) if supervisor or conf.get("restart_supervisor_on_update"): - restart_supervisor_processes(bench_path=self.name, web_workers=web) + restart_supervisor_processes(bench_path=self.name, web_workers=web, _raise=_raise) if systemd and conf.get("restart_systemd_on_update"): - restart_systemd_processes(bench_path=self.name, web_workers=web) + restart_systemd_processes(bench_path=self.name, web_workers=web, _raise=_raise) def get_installed_apps(self) -> List: """Returns list of installed apps on bench, not in excluded_apps.txt""" diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 62f3e5967..353526d55 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -276,7 +276,7 @@ def patch_sites(bench_path="."): raise PatchError -def restart_supervisor_processes(bench_path=".", web_workers=False): +def restart_supervisor_processes(bench_path=".", web_workers=False, _raise=False): from bench.bench import Bench bench = Bench(bench_path) @@ -285,7 +285,7 @@ def restart_supervisor_processes(bench_path=".", web_workers=False): bench_name = get_bench_name(bench_path) if cmd: - bench.run(cmd) + bench.run(cmd, _raise=_raise) else: sudo = "" @@ -312,18 +312,22 @@ def restart_supervisor_processes(bench_path=".", web_workers=False): else: group = "frappe:" - bench.run(f"{sudo}supervisorctl restart {group}") + failure = bench.run(f"{sudo}supervisorctl restart {group}", _raise=_raise) + if failure: + log("restarting supervisor failed. Use `bench restart` to retry.", level=3) -def restart_systemd_processes(bench_path=".", web_workers=False): +def restart_systemd_processes(bench_path=".", web_workers=False, _raise=True): bench_name = get_bench_name(bench_path) exec_cmd( f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut" - " -d= -f2)" + " -d= -f2)", + _raise=_raise, ) exec_cmd( f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |" - " cut -d= -f2)" + " cut -d= -f2)", + _raise=_raise, ) From 0234c080c83f0fa4e69e7bdadfafb1576104d243 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 12 Dec 2022 21:48:38 +0530 Subject: [PATCH 18/54] ci: run tests against current hotfix branches (#1402) - If this isn't dont then we have no way of knowing until frappe/erpnext are actually released. - Skip asset building where it doesn't matter --- bench/tests/test_base.py | 4 +--- bench/tests/test_init.py | 12 ++++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index 9ca23de44..efc1e81f8 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -15,12 +15,10 @@ PYTHON_VER = sys.version_info -FRAPPE_BRANCH = "version-12" +FRAPPE_BRANCH = "version-13-hotfix" if PYTHON_VER.major == 3: if PYTHON_VER.minor >= 10: FRAPPE_BRANCH = "develop" - if 7 >= PYTHON_VER.minor >= 9: - FRAPPE_BRANCH = "version-13" class TestBenchBase(unittest.TestCase): diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 622ffc87d..b0a871a4a 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -46,7 +46,7 @@ def basic(self): def test_multiple_benches(self): for bench_name in ("test-bench-1", "test-bench-2"): - self.init_bench(bench_name) + self.init_bench(bench_name, skip_assets=True) self.assert_common_site_config( "test-bench-1", @@ -96,7 +96,7 @@ def test_new_site(self): self.assertTrue(site_config[key]) def test_get_app(self): - self.init_bench("test-bench") + self.init_bench("test-bench", skip_assets=True) bench_path = os.path.join(self.benches_path, "test-bench") exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --skip-assets", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) @@ -108,7 +108,7 @@ def test_get_app(self): @unittest.skipIf(FRAPPE_BRANCH != "develop", "only for develop branch") def test_get_app_resolve_deps(self): FRAPPE_APP = "healthcare" - self.init_bench("test-bench") + self.init_bench("test-bench", skip_assets=True) bench_path = os.path.join(self.benches_path, "test-bench") exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps --skip-assets", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP))) @@ -126,7 +126,7 @@ def test_install_app(self): site_name = "install-app.test" bench_path = os.path.join(self.benches_path, "test-bench") - self.init_bench(bench_name) + self.init_bench(bench_name, skip_assets=True) exec_cmd( f"bench get-app {TEST_FRAPPE_APP} --branch master --skip-assets", cwd=bench_path ) @@ -154,7 +154,7 @@ def test_install_app(self): self.assertTrue(TEST_FRAPPE_APP in app_installed_on_site) def test_remove_app(self): - self.init_bench("test-bench") + self.init_bench("test-bench", skip_assets=True) bench_path = os.path.join(self.benches_path, "test-bench") exec_cmd( @@ -172,7 +172,7 @@ def test_remove_app(self): self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) def test_switch_to_branch(self): - self.init_bench("test-bench") + self.init_bench("test-bench", skip_assets=True) bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") From 931377727651da00143b062c1ce20af5adae951c Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Tue, 13 Dec 2022 13:18:09 +0530 Subject: [PATCH 19/54] chore: Update Readme and changed password generated method Co-authored-by: @ankush --- README.md | 69 ++++++++++++++++++++++++++++++++++++++++++------- easy-install.py | 19 +++++++++----- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 16f61c88b..04c87349c 100755 --- a/README.md +++ b/README.md @@ -31,17 +31,21 @@ Bench is a command-line utility that helps you to install, update, and manage mu ## Table of Contents - - [Installation](#installation) +- [Table of Contents](#table-of-contents) +- [Installation](#installation) - [Containerized Installation](#containerized-installation) + - [Easy Install Script](#easy-install-script) + - [Setup](#setup) + - [Arguments](#arguments) + - [Troubleshooting](#troubleshooting) - [Manual Installation](#manual-installation) - - [Usage](#basic-usage) - - [Custom Bench commands](#custom-bench-commands) - - [Bench Manager](#bench-manager) - - [Guides](#guides) - - [Resources](#resources) - - [Development](#development) - - [Releases](#releases) - - [License](#license) +- [Basic Usage](#basic-usage) +- [Custom Bench Commands](#custom-bench-commands) +- [Guides](#guides) +- [Resources](#resources) +- [Development](#development) +- [Releases](#releases) +- [License](#license) ## Installation @@ -71,6 +75,53 @@ $ cd frappe_docker A quick setup guide for both the environments can be found below. For more details, check out the [Frappe/ERPNext Docker Repository](https://github.com/frappe/frappe_docker). +### Easy Install Script + +The Easy Install script should get you going with a Frappe/ERPNext setup with minimal manual intervention and effort. + +This script uses Docker with the [Frappe/ERPNext Docker Repository](https://github.com/frappe/frappe_docker) and can be used for both Development setup and Production setup. + +#### Setup + +Download the Easy Install script and execute it: + +```sh +$ wget https://raw.githubusercontent.com/frappe/bench/develop/easy-install.py +$ python3 install.py --prod +``` + +This script will install docker on your system and will fetch the required containers, setup bench and a default ERPNext instance. + +The script will generate MySQL root password and an Administrator password for the Frappe/ERPNext instance, which will then be saved under `$HOME/passwords.txt` of the user used to setup the instance. +It will also generate a new compose file under `$HOME/-compose.yml`. + +When the setup is complete, you will be able to access the system at `http://`, wherein you can use the Administrator password to login. + +#### Arguments + +Here are the arguments for the easy-install script + +```txt +usage: easy-install.py [-h] [-p] [-d] [-s SITENAME] [-n PROJECT] [--email EMAIL] + +Install Frappe with Docker + +options: + -h, --help show this help message and exit + -p, --prod Setup Production System + -d, --dev Setup Development System + -s SITENAME, --sitename SITENAME + The Site Name for your production site + -n PROJECT, --project PROJECT + Project Name + --email EMAIL Add email for the SSL. +``` + +#### Troubleshooting + +In case the setup fails, the log file is saved under `$HOME/easy-install.log`. You may then + +- Create an Issue in this repository with the log file attached. ### Manual Installation diff --git a/easy-install.py b/easy-install.py index fb7424625..8334792da 100644 --- a/easy-install.py +++ b/easy-install.py @@ -14,7 +14,7 @@ logging.basicConfig( filename="easy-install.log", filemode="w", - format="%(levelname)s - %(message)s", + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO, ) @@ -91,7 +91,14 @@ def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) def generate_pass(length: int = 12) -> str: - return sha224(repr(time.time()).encode()).hexdigest()[:length] + """Generate random hash using best available randomness source.""" + import math + import secrets + + if not length: + length = 56 + + return secrets.token_hex(math.ceil(length / 2))[:length] def check_repo_exists() -> bool: @@ -108,11 +115,11 @@ def setup_prod(project: str, sitename: str, email: str) -> None: "\nPlease refer to .example.env file in the frappe_docker folder to know which keys to set\n\n", level=3, ) + admin_pass = generate_pass() + db_pass = generate_pass(9) with open(compose_file_name, "w") as f: # Writing to compose file if not os.path.exists(os.path.join(docker_repo_path, ".env")): - admin_pass = generate_pass() - db_pass = generate_pass(9) write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) cprint( "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", @@ -121,8 +128,8 @@ def setup_prod(project: str, sitename: str, email: str) -> None: with open( os.path.join(os.path.expanduser("~"), "passwords.txt"), "w" ) as en: - en.writelines(f"Administrator:{admin_pass}\n") - en.writelines(f"MariaDB Root Password:{db_pass}\n") + en.writelines(f"ADMINISTRATOR_PASSWORD={admin_pass}\n") + en.writelines(f"MARIADB_ROOT_PASSWORD={db_pass}\n") try: # TODO: Include flags for non-https and non-erpnext installation subprocess.run( From e76c7dccf57c122064dafc574b1dfa6f3d13493e Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Wed, 14 Dec 2022 17:42:24 +0530 Subject: [PATCH 20/54] feat: add GHA tests chore: added reading from env when rerunning script - Makes current site as default --- .github/workflows/easy-install.yml | 27 +++++++++++++++++++++++++++ easy-install.py | 20 +++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/easy-install.yml diff --git a/.github/workflows/easy-install.yml b/.github/workflows/easy-install.yml new file mode 100644 index 000000000..9456a364a --- /dev/null +++ b/.github/workflows/easy-install.yml @@ -0,0 +1,27 @@ +name: 'Easy Install Test' + +on: + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +jobs: + easy-install-setup: + runs-on: ubuntu-latest + timeout-minutes: 60 + + name: Easy Install Test + steps: + - uses: actions/checkout@v3 + - run: | + python3 ${GITHUB_WORKSPACE}/easy-install.py -p -n actions_test --email test@frappe.io + docker compose -p actions_test exec backend bench version --format json + docker compose -p actions_test exec backend bench --site site1.local list-apps --format json + result=$(curl -sk https://127.0.0.1/api/method/ping | jq -r ."message") + if [[ "$result" == "pong" ]]; then echo "New instance works fine"; else exit 1; fi + docker compose -p actions_test down + docker volume prune -f \ No newline at end of file diff --git a/easy-install.py b/easy-install.py index 8334792da..6faf5c135 100644 --- a/easy-install.py +++ b/easy-install.py @@ -59,9 +59,9 @@ def clone_frappe_docker_repo() -> None: logging.error("Download and unzip failed", exc_info=True) cprint("\nCloning frappe_docker Failed\n\n", "[ERROR]: ", e, level=1) -def get_latest_version(dir) -> Dict: +def get_from_env(dir,file) -> Dict: env_vars ={} - with open(os.path.join(dir,"example.env")) as f: + with open(os.path.join(dir,file)) as f: for line in f: if line.startswith('#') or not line.strip(): continue @@ -69,9 +69,10 @@ def get_latest_version(dir) -> Dict: env_vars[key] = value return env_vars + def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) -> None: site_name = site or "" - example_env = get_latest_version(wd) + example_env = get_from_env(wd,"example.env") with open(os.path.join(wd, ".env"), "w") as f: f.writelines( [ @@ -115,11 +116,13 @@ def setup_prod(project: str, sitename: str, email: str) -> None: "\nPlease refer to .example.env file in the frappe_docker folder to know which keys to set\n\n", level=3, ) - admin_pass = generate_pass() - db_pass = generate_pass(9) + admin_pass = "" + db_pass = "" with open(compose_file_name, "w") as f: # Writing to compose file if not os.path.exists(os.path.join(docker_repo_path, ".env")): + admin_pass = generate_pass() + db_pass = generate_pass(9) write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) cprint( "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", @@ -130,6 +133,10 @@ def setup_prod(project: str, sitename: str, email: str) -> None: ) as en: en.writelines(f"ADMINISTRATOR_PASSWORD={admin_pass}\n") en.writelines(f"MARIADB_ROOT_PASSWORD={db_pass}\n") + else: + env = get_from_env(docker_repo_path,".env") + admin_pass = env['SITE_ADMIN_PASS'] + db_pass = env['DB_PASSWORD'] try: # TODO: Include flags for non-https and non-erpnext installation subprocess.run( @@ -182,6 +189,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: except Exception as e: logging.error("Prod docker-compose failed", exc_info=True) cprint(" Docker Compose failed, please check the container logs\n", e) + sys.exit(1) cprint(f"\nCreating site: {sitename} \n", level=3) @@ -203,6 +211,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: admin_pass, "--install-app", "erpnext", + "--set-default" ], check=True, ) @@ -210,6 +219,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: except Exception as e: logging.error("Bench site creation failed", exc_info=True) cprint("Bench Site creation failed\n", e) + sys.exit(1) else: install_docker() clone_frappe_docker_repo() From 687044f1238f71528d6f8630148314cc6da7032e Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 15 Dec 2022 16:50:19 +0530 Subject: [PATCH 21/54] ci: add concurrency group --- .github/workflows/ci.yml | 4 ++++ .github/workflows/easy-install.yml | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1040df3a0..91e3dffda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,10 @@ on: push: branches: [ develop ] +concurrency: + group: ci-develop-${{ github.event_name }}-${{ github.event.number }} + cancel-in-progress: true + permissions: contents: read diff --git a/.github/workflows/easy-install.yml b/.github/workflows/easy-install.yml index 9456a364a..85c095c89 100644 --- a/.github/workflows/easy-install.yml +++ b/.github/workflows/easy-install.yml @@ -1,10 +1,14 @@ -name: 'Easy Install Test' +name: "Easy Install Test" on: pull_request: workflow_dispatch: push: - branches: [ develop ] + branches: [develop] + +concurrency: + group: easy-install-develop-${{ github.event_name }}-${{ github.event.number }} + cancel-in-progress: true permissions: contents: read @@ -17,11 +21,12 @@ jobs: name: Easy Install Test steps: - uses: actions/checkout@v3 - - run: | + - name: Perform production easy install + run: | python3 ${GITHUB_WORKSPACE}/easy-install.py -p -n actions_test --email test@frappe.io docker compose -p actions_test exec backend bench version --format json docker compose -p actions_test exec backend bench --site site1.local list-apps --format json result=$(curl -sk https://127.0.0.1/api/method/ping | jq -r ."message") if [[ "$result" == "pong" ]]; then echo "New instance works fine"; else exit 1; fi docker compose -p actions_test down - docker volume prune -f \ No newline at end of file + docker volume prune -f From 80b58d9999bfc8b94781a646231d1e4ebfd322b5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 15 Dec 2022 17:00:04 +0530 Subject: [PATCH 22/54] chore: typo Co-Authored-By: Athul Cyriac Ajay --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 04c87349c..840bb9db5 100755 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Download the Easy Install script and execute it: ```sh $ wget https://raw.githubusercontent.com/frappe/bench/develop/easy-install.py -$ python3 install.py --prod +$ python3 easy-install.py --prod ``` This script will install docker on your system and will fetch the required containers, setup bench and a default ERPNext instance. From a987c1e9aed4b14e0b12100f35408ebababd7dee Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 16 Dec 2022 17:32:45 +0530 Subject: [PATCH 23/54] style: chmod+x, format, space -> tabs --- easy-install.py | 535 ++++++++++++++++++++++++------------------------ 1 file changed, 266 insertions(+), 269 deletions(-) mode change 100644 => 100755 easy-install.py diff --git a/easy-install.py b/easy-install.py old mode 100644 new mode 100755 index 6faf5c135..82fc17385 --- a/easy-install.py +++ b/easy-install.py @@ -1,94 +1,95 @@ #!/usr/bin/env python3 + import argparse -import subprocess +import logging import os +import platform +import subprocess import sys import time import urllib.request -import logging -import platform -from shutil import which, unpack_archive, move -from hashlib import sha224 +from shutil import move, unpack_archive, which from typing import Dict logging.basicConfig( - filename="easy-install.log", - filemode="w", - format="%(asctime)s - %(levelname)s - %(message)s", - level=logging.INFO, + filename="easy-install.log", + filemode="w", + format="%(asctime)s - %(levelname)s - %(message)s", + level=logging.INFO, ) def cprint(*args, level: int = 1): - """ - logs colorful messages - level = 1 : RED - level = 2 : GREEN - level = 3 : YELLOW + """ + logs colorful messages + level = 1 : RED + level = 2 : GREEN + level = 3 : YELLOW - default level = 1 - """ - CRED = "\033[31m" - CGRN = "\33[92m" - CYLW = "\33[93m" - reset = "\033[0m" - message = " ".join(map(str, args)) - if level == 1: - print(CRED, message, reset) - if level == 2: - print(CGRN, message, reset) - if level == 3: - print(CYLW, message, reset) + default level = 1 + """ + CRED = "\033[31m" + CGRN = "\33[92m" + CYLW = "\33[93m" + reset = "\033[0m" + message = " ".join(map(str, args)) + if level == 1: + print(CRED, message, reset) + if level == 2: + print(CGRN, message, reset) + if level == 3: + print(CYLW, message, reset) def clone_frappe_docker_repo() -> None: - try: - urllib.request.urlretrieve( - "https://github.com/frappe/frappe_docker/archive/refs/heads/main.zip", - "frappe_docker.zip", - ) - logging.info("Downloaded frappe_docker zip file from GitHub") - unpack_archive( - "frappe_docker.zip", "." - ) # Unzipping the frappe_docker.zip creates a folder "frappe_docker-main" - move("frappe_docker-main", "frappe_docker") - logging.info("Unzipped and Renamed frappe_docker") - os.remove("frappe_docker.zip") - logging.info("Removed the downloaded zip file") - except Exception as e: - logging.error("Download and unzip failed", exc_info=True) - cprint("\nCloning frappe_docker Failed\n\n", "[ERROR]: ", e, level=1) + try: + urllib.request.urlretrieve( + "https://github.com/frappe/frappe_docker/archive/refs/heads/main.zip", + "frappe_docker.zip", + ) + logging.info("Downloaded frappe_docker zip file from GitHub") + unpack_archive( + "frappe_docker.zip", "." + ) # Unzipping the frappe_docker.zip creates a folder "frappe_docker-main" + move("frappe_docker-main", "frappe_docker") + logging.info("Unzipped and Renamed frappe_docker") + os.remove("frappe_docker.zip") + logging.info("Removed the downloaded zip file") + except Exception as e: + logging.error("Download and unzip failed", exc_info=True) + cprint("\nCloning frappe_docker Failed\n\n", "[ERROR]: ", e, level=1) + -def get_from_env(dir,file) -> Dict: - env_vars ={} - with open(os.path.join(dir,file)) as f: - for line in f: - if line.startswith('#') or not line.strip(): - continue - key, value = line.strip().split('=', 1) - env_vars[key] = value - return env_vars +def get_from_env(dir, file) -> Dict: + env_vars = {} + with open(os.path.join(dir, file)) as f: + for line in f: + if line.startswith("#") or not line.strip(): + continue + key, value = line.strip().split("=", 1) + env_vars[key] = value + return env_vars def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) -> None: - site_name = site or "" - example_env = get_from_env(wd,"example.env") - with open(os.path.join(wd, ".env"), "w") as f: - f.writelines( - [ - f"FRAPPE_VERSION={example_env['FRAPPE_VERSION']}\n", # Defaults to latest version of Frappe - f"ERPNEXT_VERSION={example_env['ERPNEXT_VERSION']}\n", # defaults to latest version of ERPNext - f"DB_PASSWORD={db_pass}\n", - "DB_HOST=db\n", - "DB_PORT=3306\n", - "REDIS_CACHE=redis-cache:6379\n", - "REDIS_QUEUE=redis-queue:6379\n", - "REDIS_SOCKETIO=redis-socketio:6379\n", - f"LETSENCRYPT_EMAIL={email}\n", - f"FRAPPE_SITE_NAME_HEADER={site_name}\n", - f"SITE_ADMIN_PASS={admin_pass}", - ] - ) + site_name = site or "" + example_env = get_from_env(wd, "example.env") + with open(os.path.join(wd, ".env"), "w") as f: + f.writelines( + [ + f"FRAPPE_VERSION={example_env['FRAPPE_VERSION']}\n", # Defaults to latest version of Frappe + f"ERPNEXT_VERSION={example_env['ERPNEXT_VERSION']}\n", # defaults to latest version of ERPNext + f"DB_PASSWORD={db_pass}\n", + "DB_HOST=db\n", + "DB_PORT=3306\n", + "REDIS_CACHE=redis-cache:6379\n", + "REDIS_QUEUE=redis-queue:6379\n", + "REDIS_SOCKETIO=redis-socketio:6379\n", + f"LETSENCRYPT_EMAIL={email}\n", + f"FRAPPE_SITE_NAME_HEADER={site_name}\n", + f"SITE_ADMIN_PASS={admin_pass}", + ] + ) def generate_pass(length: int = 12) -> str: @@ -103,221 +104,217 @@ def generate_pass(length: int = 12) -> str: def check_repo_exists() -> bool: - return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) + return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) def setup_prod(project: str, sitename: str, email: str) -> None: - if check_repo_exists(): - compose_file_name = os.path.join( - os.path.expanduser("~"), f"{project}-compose.yml" - ) - docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") - cprint( - "\nPlease refer to .example.env file in the frappe_docker folder to know which keys to set\n\n", - level=3, - ) - admin_pass = "" - db_pass = "" - with open(compose_file_name, "w") as f: - # Writing to compose file - if not os.path.exists(os.path.join(docker_repo_path, ".env")): - admin_pass = generate_pass() - db_pass = generate_pass(9) - write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) - cprint( - "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", - level=3, - ) - with open( - os.path.join(os.path.expanduser("~"), "passwords.txt"), "w" - ) as en: - en.writelines(f"ADMINISTRATOR_PASSWORD={admin_pass}\n") - en.writelines(f"MARIADB_ROOT_PASSWORD={db_pass}\n") - else: - env = get_from_env(docker_repo_path,".env") - admin_pass = env['SITE_ADMIN_PASS'] - db_pass = env['DB_PASSWORD'] - try: - # TODO: Include flags for non-https and non-erpnext installation - subprocess.run( - [ - which("docker"), - "compose", - "--project-name", - project, - "-f", - "compose.yaml", - "-f", - "overrides/compose.mariadb.yaml", - "-f", - "overrides/compose.redis.yaml", - # "-f", "overrides/compose.noproxy.yaml", TODO: Add support for local proxying without HTTPs - "-f", - "overrides/compose.erpnext.yaml", - "-f", - "overrides/compose.https.yaml", - "--env-file", - ".env", - "config", - ], - cwd=docker_repo_path, - stdout=f, - check=True, - ) + if check_repo_exists(): + compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") + docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") + cprint( + "\nPlease refer to .example.env file in the frappe_docker folder to know which keys to set\n\n", + level=3, + ) + admin_pass = "" + db_pass = "" + with open(compose_file_name, "w") as f: + # Writing to compose file + if not os.path.exists(os.path.join(docker_repo_path, ".env")): + admin_pass = generate_pass() + db_pass = generate_pass(9) + write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) + cprint( + "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", + level=3, + ) + with open(os.path.join(os.path.expanduser("~"), "passwords.txt"), "w") as en: + en.writelines(f"ADMINISTRATOR_PASSWORD={admin_pass}\n") + en.writelines(f"MARIADB_ROOT_PASSWORD={db_pass}\n") + else: + env = get_from_env(docker_repo_path, ".env") + admin_pass = env["SITE_ADMIN_PASS"] + db_pass = env["DB_PASSWORD"] + try: + # TODO: Include flags for non-https and non-erpnext installation + subprocess.run( + [ + which("docker"), + "compose", + "--project-name", + project, + "-f", + "compose.yaml", + "-f", + "overrides/compose.mariadb.yaml", + "-f", + "overrides/compose.redis.yaml", + # "-f", "overrides/compose.noproxy.yaml", TODO: Add support for local proxying without HTTPs + "-f", + "overrides/compose.erpnext.yaml", + "-f", + "overrides/compose.https.yaml", + "--env-file", + ".env", + "config", + ], + cwd=docker_repo_path, + stdout=f, + check=True, + ) - except Exception as e: - logging.error("Docker Compose generation failed", exc_info=True) - cprint("\nGenerating Compose File failed\n") - sys.exit(1) - try: - # Starting with generated compose file - subprocess.run( - [ - which("docker"), - "compose", - "-p", - project, - "-f", - compose_file_name, - "up", - "-d", - ], - check=True, - ) - logging.info(f"Docker Compose file generated at ~/{project}-compose.yml") + except Exception: + logging.error("Docker Compose generation failed", exc_info=True) + cprint("\nGenerating Compose File failed\n") + sys.exit(1) + try: + # Starting with generated compose file + subprocess.run( + [ + which("docker"), + "compose", + "-p", + project, + "-f", + compose_file_name, + "up", + "-d", + ], + check=True, + ) + logging.info(f"Docker Compose file generated at ~/{project}-compose.yml") - except Exception as e: - logging.error("Prod docker-compose failed", exc_info=True) - cprint(" Docker Compose failed, please check the container logs\n", e) - sys.exit(1) + except Exception as e: + logging.error("Prod docker-compose failed", exc_info=True) + cprint(" Docker Compose failed, please check the container logs\n", e) + sys.exit(1) - cprint(f"\nCreating site: {sitename} \n", level=3) + cprint(f"\nCreating site: {sitename} \n", level=3) - try: - subprocess.run( - [ - which("docker"), - "compose", - "-p", - project, - "exec", - "backend", - "bench", - "new-site", - sitename, - "--db-root-password", - db_pass, - "--admin-password", - admin_pass, - "--install-app", - "erpnext", - "--set-default" - ], - check=True, - ) - logging.info("New site creation completed") - except Exception as e: - logging.error("Bench site creation failed", exc_info=True) - cprint("Bench Site creation failed\n", e) - sys.exit(1) - else: - install_docker() - clone_frappe_docker_repo() - setup_prod(project, sitename, email) # Recursive + try: + subprocess.run( + [ + which("docker"), + "compose", + "-p", + project, + "exec", + "backend", + "bench", + "new-site", + sitename, + "--db-root-password", + db_pass, + "--admin-password", + admin_pass, + "--install-app", + "erpnext", + "--set-default", + ], + check=True, + ) + logging.info("New site creation completed") + except Exception as e: + logging.error("Bench site creation failed", exc_info=True) + cprint("Bench Site creation failed\n", e) + sys.exit(1) + else: + install_docker() + clone_frappe_docker_repo() + setup_prod(project, sitename, email) # Recursive def setup_dev_instance(project: str): - if check_repo_exists(): - try: - subprocess.run( - [ - "docker", - "compose", - "-f", - "devcontainer-example/docker-compose.yml", - "--project-name", - project, - "up", - "-d", - ], - cwd=os.path.join(os.getcwd(), "frappe_docker"), - check=True, - ) - cprint( - "Please go through the Development Documentation: https://github.com/frappe/frappe_docker/tree/main/development to fully complete the setup.", - level=2, - ) - logging.info("Development Setup completed") - except Exception as e: - logging.error("Dev Environment setup failed", exc_info=True) - cprint("Setting Up Development Environment Failed\n", e) - else: - install_docker() - clone_frappe_docker_repo() - setup_dev_instance(project) # Recursion on goes brrrr + if check_repo_exists(): + try: + subprocess.run( + [ + "docker", + "compose", + "-f", + "devcontainer-example/docker-compose.yml", + "--project-name", + project, + "up", + "-d", + ], + cwd=os.path.join(os.getcwd(), "frappe_docker"), + check=True, + ) + cprint( + "Please go through the Development Documentation: https://github.com/frappe/frappe_docker/tree/main/development to fully complete the setup.", + level=2, + ) + logging.info("Development Setup completed") + except Exception as e: + logging.error("Dev Environment setup failed", exc_info=True) + cprint("Setting Up Development Environment Failed\n", e) + else: + install_docker() + clone_frappe_docker_repo() + setup_dev_instance(project) # Recursion on goes brrrr def install_docker(): - if which("docker") is not None: - return - cprint("Docker is not installed, Installing Docker...", level=3) - logging.info("Docker not found, installing Docker") - if platform.system() == "Darwin" or platform.system() == "Windows": - print( - f""" + if which("docker") is not None: + return + cprint("Docker is not installed, Installing Docker...", level=3) + logging.info("Docker not found, installing Docker") + if platform.system() == "Darwin" or platform.system() == "Windows": + print( + f""" This script doesn't install Docker on {"Mac" if platform.system()=="Darwin" else "Windows"}. Please go through the Docker Installation docs for your system and run this script again""" - ) - logging.debug("Docker setup failed due to platform is not Linux") - sys.exit(1) - try: - ps = subprocess.run( - ["curl", "-fsSL", "https://get.docker.com"], - capture_output=True, - check=True, - ) - subprocess.run(["/bin/bash"], input=ps.stdout, capture_output=True) - subprocess.run( - ["sudo", "usermod", "-aG", "docker", str(os.getenv("USER"))], check=True - ) - cprint("Waiting Docker to start", level=3) - time.sleep(10) - subprocess.run(["sudo", "systemctl", "restart", "docker.service"], check=True) - except Exception as e: - logging.error("Installing Docker failed", exc_info=True) - cprint("Failed to Install Docker\n", e) - cprint("\n Try Installing Docker Manually and re-run this script again\n") - sys.exit(1) + ) + logging.debug("Docker setup failed due to platform is not Linux") + sys.exit(1) + try: + ps = subprocess.run( + ["curl", "-fsSL", "https://get.docker.com"], + capture_output=True, + check=True, + ) + subprocess.run(["/bin/bash"], input=ps.stdout, capture_output=True) + subprocess.run( + ["sudo", "usermod", "-aG", "docker", str(os.getenv("USER"))], check=True + ) + cprint("Waiting Docker to start", level=3) + time.sleep(10) + subprocess.run(["sudo", "systemctl", "restart", "docker.service"], check=True) + except Exception as e: + logging.error("Installing Docker failed", exc_info=True) + cprint("Failed to Install Docker\n", e) + cprint("\n Try Installing Docker Manually and re-run this script again\n") + sys.exit(1) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Install Frappe with Docker") - parser.add_argument( - "-p", "--prod", help="Setup Production System", action="store_true" - ) - parser.add_argument( - "-d", "--dev", help="Setup Development System", action="store_true" - ) - parser.add_argument( - "-s", - "--sitename", - help="The Site Name for your production site", - default="site1.local", - ) - parser.add_argument("-n", "--project", help="Project Name", default="frappe") - parser.add_argument( - "--email", help="Add email for the SSL.", required="--prod" in sys.argv - ) - args = parser.parse_args() - if args.dev: - cprint("\nSetting Up Development Instance\n", level=2) - logging.info("Running Development Setup") - setup_dev_instance(args.project) - elif args.prod: - cprint("\nSetting Up Production Instance\n", level=2) - logging.info("Running Production Setup") - if "example.com" in args.email: - cprint("Emails with example.com not acceptable", level=1) - sys.exit(1) - setup_prod(args.project, args.sitename, args.email) \ No newline at end of file + parser = argparse.ArgumentParser(description="Install Frappe with Docker") + parser.add_argument( + "-p", "--prod", help="Setup Production System", action="store_true" + ) + parser.add_argument( + "-d", "--dev", help="Setup Development System", action="store_true" + ) + parser.add_argument( + "-s", + "--sitename", + help="The Site Name for your production site", + default="site1.local", + ) + parser.add_argument("-n", "--project", help="Project Name", default="frappe") + parser.add_argument( + "--email", help="Add email for the SSL.", required="--prod" in sys.argv + ) + args = parser.parse_args() + if args.dev: + cprint("\nSetting Up Development Instance\n", level=2) + logging.info("Running Development Setup") + setup_dev_instance(args.project) + elif args.prod: + cprint("\nSetting Up Production Instance\n", level=2) + logging.info("Running Production Setup") + if "example.com" in args.email: + cprint("Emails with example.com not acceptable", level=1) + sys.exit(1) + setup_prod(args.project, args.sitename, args.email) From 228aeaf2fdf0205294825a40c6752b2c8abf6d7a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 16 Dec 2022 17:34:26 +0530 Subject: [PATCH 24/54] fix: print help when no args passed [skip ci] --- easy-install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easy-install.py b/easy-install.py index 82fc17385..26d5ab720 100755 --- a/easy-install.py +++ b/easy-install.py @@ -318,3 +318,5 @@ def install_docker(): cprint("Emails with example.com not acceptable", level=1) sys.exit(1) setup_prod(args.project, args.sitename, args.email) + else: + parser.print_help() From 8903649159676882bd9e970ecefd44c969856f05 Mon Sep 17 00:00:00 2001 From: jiangying Date: Sat, 31 Dec 2022 16:20:21 +0800 Subject: [PATCH 25/54] chore: typo in readme (#1407) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 840bb9db5..5e4edcdd0 100755 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ The setup for each of these installations can be achieved in multiple ways: - [Containerized Installation](#containerized-installation) - [Manual Installation](#manual-installation) -We recommend using either the Docker Installation to setup a Production Environment. For Development, you may choose either of the two methods to setup an instance. +We recommend using Docker Installation to setup a Production Environment. For Development, you may choose either of the two methods to setup an instance. Otherwise, if you are looking to evaluate Frappe apps without hassle of hosting, you can try them [on frappecloud.com](https://frappecloud.com/). From ba853c943b375e5ec32c917aada918a13e0fef03 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 2 Jan 2023 15:47:05 +0530 Subject: [PATCH 26/54] chore: incorrect license identifiers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cab5236a2..2d535b1d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", - "License :: OSI Approved :: GNU Affero General Public License v3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Natural Language :: English", "Operating System :: MacOS", "Operating System :: OS Independent", From 23eede5fd33501f58770d32ac2f4ba1793339b82 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 2 Jan 2023 16:03:33 +0530 Subject: [PATCH 27/54] fix: version check backward compatibility (#1409) This code wasn't triggering because VersionNotFound exception gets thrown before it ever reaches to this point. --- bench/utils/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils/app.py b/bench/utils/app.py index 5541b548e..75891d5bf 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -284,7 +284,7 @@ def get_current_version(app, bench_path="."): with open(init_path) as f: current_version = get_version_from_string(f.read()) - except AttributeError: + except (AttributeError, VersionNotFound): # backward compatibility with open(setup_path) as f: current_version = get_version_from_string(f.read(), field="version") From 34197056ea7de6735ea0fd8fc63b18093b3f634b Mon Sep 17 00:00:00 2001 From: Himanshu Shivhare Date: Fri, 6 Jan 2023 15:18:47 +0530 Subject: [PATCH 28/54] chore: typo/readability fix (#1410) --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5e4edcdd0..a5b9db56a 100755 --- a/README.md +++ b/README.md @@ -107,14 +107,12 @@ usage: easy-install.py [-h] [-p] [-d] [-s SITENAME] [-n PROJECT] [--email EMAIL] Install Frappe with Docker options: - -h, --help show this help message and exit - -p, --prod Setup Production System - -d, --dev Setup Development System - -s SITENAME, --sitename SITENAME - The Site Name for your production site - -n PROJECT, --project PROJECT - Project Name - --email EMAIL Add email for the SSL. + -h, --help show this help message and exit + -p, --prod Setup Production System + -d, --dev Setup Development System + -s SITENAME, --sitename SITENAME The Site Name for your production site + -n PROJECT, --project PROJECT Project Name + --email EMAIL Add email for the SSL. ``` #### Troubleshooting From b3ad10b9e63ad2e966cdc89bc65f2a79be08cc7e Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 16 Jan 2023 16:24:56 +0530 Subject: [PATCH 29/54] ci: fix easy-install.py test (#1408) * ci: fix easy-install.py test related to https://github.com/frappe/frappe_docker/pull/1032 * ci: fix easy-install.py patched bench removed and frappe-bench installed --no-mariadb-socket required for new-site * fix: remove frappe version from .env * feat(easy-install): option to set version Co-authored-by: Ankush Menat --- easy-install.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/easy-install.py b/easy-install.py index 26d5ab720..33bd56a5c 100755 --- a/easy-install.py +++ b/easy-install.py @@ -71,14 +71,21 @@ def get_from_env(dir, file) -> Dict: return env_vars -def write_to_env(wd: str, site: str, db_pass: str, admin_pass: str, email: str) -> None: +def write_to_env( + wd: str, + site: str, + db_pass: str, + admin_pass: str, + email: str, + erpnext_version: str = None, +) -> None: site_name = site or "" example_env = get_from_env(wd, "example.env") + erpnext_version = erpnext_version or example_env['ERPNEXT_VERSION'] with open(os.path.join(wd, ".env"), "w") as f: f.writelines( [ - f"FRAPPE_VERSION={example_env['FRAPPE_VERSION']}\n", # Defaults to latest version of Frappe - f"ERPNEXT_VERSION={example_env['ERPNEXT_VERSION']}\n", # defaults to latest version of ERPNext + f"ERPNEXT_VERSION={erpnext_version}\n", # defaults to latest version of ERPNext f"DB_PASSWORD={db_pass}\n", "DB_HOST=db\n", "DB_PORT=3306\n", @@ -107,7 +114,7 @@ def check_repo_exists() -> bool: return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) -def setup_prod(project: str, sitename: str, email: str) -> None: +def setup_prod(project: str, sitename: str, email: str, version: str = None) -> None: if check_repo_exists(): compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") @@ -122,7 +129,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: if not os.path.exists(os.path.join(docker_repo_path, ".env")): admin_pass = generate_pass() db_pass = generate_pass(9) - write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email) + write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email, version) cprint( "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", level=3, @@ -150,8 +157,6 @@ def setup_prod(project: str, sitename: str, email: str) -> None: "overrides/compose.redis.yaml", # "-f", "overrides/compose.noproxy.yaml", TODO: Add support for local proxying without HTTPs "-f", - "overrides/compose.erpnext.yaml", - "-f", "overrides/compose.https.yaml", "--env-file", ".env", @@ -202,6 +207,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: "bench", "new-site", sitename, + "--no-mariadb-socket", "--db-root-password", db_pass, "--admin-password", @@ -220,7 +226,7 @@ def setup_prod(project: str, sitename: str, email: str) -> None: else: install_docker() clone_frappe_docker_repo() - setup_prod(project, sitename, email) # Recursive + setup_prod(project, sitename, email, version) # Recursive def setup_dev_instance(project: str): @@ -306,6 +312,7 @@ def install_docker(): parser.add_argument( "--email", help="Add email for the SSL.", required="--prod" in sys.argv ) + parser.add_argument("-v", "--version", help="ERPNext version to install, defaults to latest stable") args = parser.parse_args() if args.dev: cprint("\nSetting Up Development Instance\n", level=2) @@ -317,6 +324,6 @@ def install_docker(): if "example.com" in args.email: cprint("Emails with example.com not acceptable", level=1) sys.exit(1) - setup_prod(args.project, args.sitename, args.email) + setup_prod(args.project, args.sitename, args.email, args.version) else: parser.print_help() From e58a56e2473b9b72ac3f6cb77db2fc19831b2238 Mon Sep 17 00:00:00 2001 From: trs998 Date: Fri, 20 Jan 2023 17:37:38 +0000 Subject: [PATCH 30/54] docs: Added --email argument to install script (#1417) The install script requires the --email parameter, so added to example command line --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a5b9db56a..e523672c0 100755 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Download the Easy Install script and execute it: ```sh $ wget https://raw.githubusercontent.com/frappe/bench/develop/easy-install.py -$ python3 easy-install.py --prod +$ python3 easy-install.py --prod --email your@email.tld ``` This script will install docker on your system and will fetch the required containers, setup bench and a default ERPNext instance. From db165d1f9b0b10ca39f39d407737bd4f515d7d83 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 24 Jan 2023 15:21:24 +0530 Subject: [PATCH 31/54] fix: Setup chdir patch earlier Setup monkey patching before any other bench code decides to chdir --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 140ea6626..7f262407d 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -60,6 +60,7 @@ def execute_cmd(check_for_update=True, command: str = None, logger: Logger = Non def cli(): + setup_clear_cache() global from_command_line, bench_config, is_envvar_warn_set, verbose from_command_line = True @@ -75,7 +76,6 @@ def cli(): change_working_directory() logger = setup_logging() logger.info(command) - setup_clear_cache() bench_config = get_config(".") From 3f9360bc42ceacadced5b8e0d94be71383701090 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 24 Jan 2023 15:26:11 +0530 Subject: [PATCH 32/54] chore: Drop dead deepsource conf --- .deepsource.toml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml deleted file mode 100644 index 90b9b9bba..000000000 --- a/.deepsource.toml +++ /dev/null @@ -1,16 +0,0 @@ -version = 1 - -exclude_patterns = [ - ".*" -] - -test_patterns = [ - "bench/tests/**" -] - -[[analyzers]] -name = "python" -enabled = true -dependency_file_paths = [ - "requirements.txt" -] \ No newline at end of file From 044d646ec5b64acf95133d56cd255c642159750c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 24 Jan 2023 15:26:32 +0530 Subject: [PATCH 33/54] style: Pre-commit linters --- easy-install.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easy-install.py b/easy-install.py index 33bd56a5c..e81fb3183 100755 --- a/easy-install.py +++ b/easy-install.py @@ -81,7 +81,7 @@ def write_to_env( ) -> None: site_name = site or "" example_env = get_from_env(wd, "example.env") - erpnext_version = erpnext_version or example_env['ERPNEXT_VERSION'] + erpnext_version = erpnext_version or example_env["ERPNEXT_VERSION"] with open(os.path.join(wd, ".env"), "w") as f: f.writelines( [ @@ -312,7 +312,9 @@ def install_docker(): parser.add_argument( "--email", help="Add email for the SSL.", required="--prod" in sys.argv ) - parser.add_argument("-v", "--version", help="ERPNext version to install, defaults to latest stable") + parser.add_argument( + "-v", "--version", help="ERPNext version to install, defaults to latest stable" + ) args = parser.parse_args() if args.dev: cprint("\nSetting Up Development Instance\n", level=2) From dd77fd87fde8f303e006ad0c57986a12325ccb88 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 30 Jan 2023 16:44:29 +0530 Subject: [PATCH 34/54] build: Bump GitPython 2.1.x => 3.1.x (#1423) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d535b1d8..d1011ff18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ ] dependencies = [ "Click>=7.0", - "GitPython~=2.1.15", + "GitPython~=3.1.30", "honcho", "Jinja2~=3.0.3", "python-crontab~=2.6.0", From b9e9eac4115b74e3f6adccc2739ae2f7f8f9db1f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 30 Jan 2023 20:26:01 +0530 Subject: [PATCH 35/54] ci: Use node18 for releases --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c16bba90b..27ff45c9e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v2 with: - node-version: 14 + node-version: 18 - uses: actions/setup-python@v4 with: python-version: '3.x' From c690e8e7f53d20c2297734a0159f7b36d93a8f38 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 24 Feb 2023 15:13:36 +0530 Subject: [PATCH 36/54] fix: Handle supervisor escalation better (#1438) * fix: Give more meaningful context in subproc failures * fix: Handle supervisor escalation if no exc is raised * fix: only apply sudo if not already running as sudo --- bench/utils/__init__.py | 6 +++--- bench/utils/bench.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index b61f686e1..b9e0cced4 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -150,12 +150,12 @@ def exec_cmd(cmd, cwd=".", env=None, _raise=True): cwd_info = f"cd {cwd} && " if cwd != "." else "" cmd_log = f"{cwd_info}{cmd}" logger.debug(cmd_log) - cmd = split(cmd) - return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env) + spl_cmd = split(cmd) + return_code = subprocess.call(spl_cmd, cwd=cwd, universal_newlines=True, env=env) if return_code: logger.warning(f"{cmd_log} executed with exit code {return_code}") if _raise: - raise CommandFailedError + raise CommandFailedError from subprocess.CalledProcessError(return_code, cmd) return return_code diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 353526d55..3fe7f85b6 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -298,6 +298,12 @@ def restart_supervisor_processes(bench_path=".", web_workers=False, _raise=False sudo = "sudo " supervisor_status = get_cmd_output("sudo supervisorctl status", cwd=bench_path) + if not sudo and ( + "error: , [Errno 13] Permission denied" in supervisor_status + ): + sudo = "sudo " + supervisor_status = get_cmd_output("sudo supervisorctl status", cwd=bench_path) + if web_workers and f"{bench_name}-web:" in supervisor_status: group = f"{bench_name}-web:\t" From eba0f7a54f5d08dc91e331af3edad9f0fbf196a3 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 27 Feb 2023 10:58:33 +0530 Subject: [PATCH 37/54] fix: easy-install for letsencrypt tls (#1424) * fix: easy-install for letsencrypt tls * ci: change site for easy-install test * ci: set host header for easy-install ping test --- .github/workflows/easy-install.yml | 4 +- easy-install.py | 90 +++++++++++++++++------------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/.github/workflows/easy-install.yml b/.github/workflows/easy-install.yml index 85c095c89..9f3f36026 100644 --- a/.github/workflows/easy-install.yml +++ b/.github/workflows/easy-install.yml @@ -25,8 +25,8 @@ jobs: run: | python3 ${GITHUB_WORKSPACE}/easy-install.py -p -n actions_test --email test@frappe.io docker compose -p actions_test exec backend bench version --format json - docker compose -p actions_test exec backend bench --site site1.local list-apps --format json - result=$(curl -sk https://127.0.0.1/api/method/ping | jq -r ."message") + docker compose -p actions_test exec backend bench --site site1.localhost list-apps --format json + result=$(curl -H "Host: site1.localhost" -sk https://127.0.0.1/api/method/ping | jq -r ."message") if [[ "$result" == "pong" ]]; then echo "New instance works fine"; else exit 1; fi docker compose -p actions_test down docker volume prune -f diff --git a/easy-install.py b/easy-install.py index e81fb3183..27e565140 100755 --- a/easy-install.py +++ b/easy-install.py @@ -73,13 +73,13 @@ def get_from_env(dir, file) -> Dict: def write_to_env( wd: str, - site: str, + sites: list[str], db_pass: str, admin_pass: str, email: str, erpnext_version: str = None, ) -> None: - site_name = site or "" + quoted_sites = ",".join([f"`{site}`" for site in sites]).strip(",") example_env = get_from_env(wd, "example.env") erpnext_version = erpnext_version or example_env["ERPNEXT_VERSION"] with open(os.path.join(wd, ".env"), "w") as f: @@ -93,8 +93,8 @@ def write_to_env( "REDIS_QUEUE=redis-queue:6379\n", "REDIS_SOCKETIO=redis-socketio:6379\n", f"LETSENCRYPT_EMAIL={email}\n", - f"FRAPPE_SITE_NAME_HEADER={site_name}\n", - f"SITE_ADMIN_PASS={admin_pass}", + f"SITE_ADMIN_PASS={admin_pass}\n", + f"SITES={quoted_sites}\n", ] ) @@ -114,7 +114,7 @@ def check_repo_exists() -> bool: return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) -def setup_prod(project: str, sitename: str, email: str, version: str = None) -> None: +def setup_prod(project: str, sites: list[str], email: str, version: str = None) -> None: if check_repo_exists(): compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") @@ -129,7 +129,7 @@ def setup_prod(project: str, sitename: str, email: str, version: str = None) -> if not os.path.exists(os.path.join(docker_repo_path, ".env")): admin_pass = generate_pass() db_pass = generate_pass(9) - write_to_env(docker_repo_path, sitename, db_pass, admin_pass, email, version) + write_to_env(docker_repo_path, sites, db_pass, admin_pass, email, version) cprint( "\nA .env file is generated with basic configs. Please edit it to fit to your needs \n", level=3, @@ -193,40 +193,13 @@ def setup_prod(project: str, sitename: str, email: str, version: str = None) -> cprint(" Docker Compose failed, please check the container logs\n", e) sys.exit(1) - cprint(f"\nCreating site: {sitename} \n", level=3) + for sitename in sites: + create_site(sitename, project, db_pass, admin_pass) - try: - subprocess.run( - [ - which("docker"), - "compose", - "-p", - project, - "exec", - "backend", - "bench", - "new-site", - sitename, - "--no-mariadb-socket", - "--db-root-password", - db_pass, - "--admin-password", - admin_pass, - "--install-app", - "erpnext", - "--set-default", - ], - check=True, - ) - logging.info("New site creation completed") - except Exception as e: - logging.error("Bench site creation failed", exc_info=True) - cprint("Bench Site creation failed\n", e) - sys.exit(1) else: install_docker() clone_frappe_docker_repo() - setup_prod(project, sitename, email, version) # Recursive + setup_prod(project, sites, email, version) # Recursive def setup_dev_instance(project: str): @@ -294,6 +267,43 @@ def install_docker(): sys.exit(1) +def create_site( + sitename: str, + project: str, + db_pass: str, + admin_pass: str, +): + cprint(f"\nCreating site: {sitename} \n", level=3) + + try: + subprocess.run( + [ + which("docker"), + "compose", + "-p", + project, + "exec", + "backend", + "bench", + "new-site", + sitename, + "--no-mariadb-socket", + "--db-root-password", + db_pass, + "--admin-password", + admin_pass, + "--install-app", + "erpnext", + "--set-default", + ], + check=True, + ) + logging.info("New site creation completed") + except Exception as e: + logging.error(f"Bench site creation failed for {sitename}", exc_info=True) + cprint(f"Bench Site creation failed for {sitename}\n", e) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Install Frappe with Docker") parser.add_argument( @@ -305,8 +315,10 @@ def install_docker(): parser.add_argument( "-s", "--sitename", - help="The Site Name for your production site", - default="site1.local", + help="Site Name(s) for your production bench", + default=["site1.localhost"], + action='append', + dest='sites' ) parser.add_argument("-n", "--project", help="Project Name", default="frappe") parser.add_argument( @@ -326,6 +338,6 @@ def install_docker(): if "example.com" in args.email: cprint("Emails with example.com not acceptable", level=1) sys.exit(1) - setup_prod(args.project, args.sitename, args.email, args.version) + setup_prod(args.project, args.sites, args.email, args.version) else: parser.print_help() From 4ec73c3b92065c6c31976e201feb9188dd23836e Mon Sep 17 00:00:00 2001 From: Saif Ur Rehman Date: Mon, 27 Feb 2023 13:46:51 +0500 Subject: [PATCH 38/54] fix: add customer background workers in group (#1439) --- bench/config/templates/supervisor.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index f29c1673e..edf00cf47 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -182,12 +182,12 @@ programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-sock {% if use_rq %} [group:{{ bench_name }}-workers] -programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker +programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %} {% else %} [group:{{ bench_name }}-workers] -programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker +programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %} {% endif %} From 9fb55e6dcac40183306e134b265ded0037f4f2cf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 28 Feb 2023 10:24:41 +0530 Subject: [PATCH 39/54] chore: remove unsupported annotations these annotations dont work on python <3.9 closes https://github.com/frappe/bench/issues/1440 --- easy-install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easy-install.py b/easy-install.py index 27e565140..dfb3040df 100755 --- a/easy-install.py +++ b/easy-install.py @@ -73,7 +73,7 @@ def get_from_env(dir, file) -> Dict: def write_to_env( wd: str, - sites: list[str], + sites, db_pass: str, admin_pass: str, email: str, @@ -317,8 +317,8 @@ def create_site( "--sitename", help="Site Name(s) for your production bench", default=["site1.localhost"], - action='append', - dest='sites' + action="append", + dest="sites", ) parser.add_argument("-n", "--project", help="Project Name", default="frappe") parser.add_argument( From f7a0d281748019a3e26d40d073135df1e82cc107 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 1 Mar 2023 10:26:54 +0530 Subject: [PATCH 40/54] chore: remove unsupported annotations closes https://github.com/frappe/bench/issues/1441 --- easy-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easy-install.py b/easy-install.py index dfb3040df..f703419f3 100755 --- a/easy-install.py +++ b/easy-install.py @@ -114,7 +114,7 @@ def check_repo_exists() -> bool: return os.path.exists(os.path.join(os.getcwd(), "frappe_docker")) -def setup_prod(project: str, sites: list[str], email: str, version: str = None) -> None: +def setup_prod(project: str, sites, email: str, version: str = None) -> None: if check_repo_exists(): compose_file_name = os.path.join(os.path.expanduser("~"), f"{project}-compose.yml") docker_repo_path = os.path.join(os.getcwd(), "frappe_docker") From fd6dfc3ae77b9df84373b7e7df19e7ca29333506 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 1 Mar 2023 10:36:50 +0530 Subject: [PATCH 41/54] ci: run easy install test with lower python version (#1442) --- .github/workflows/easy-install.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/easy-install.yml b/.github/workflows/easy-install.yml index 9f3f36026..e9ccc03f0 100644 --- a/.github/workflows/easy-install.yml +++ b/.github/workflows/easy-install.yml @@ -21,6 +21,11 @@ jobs: name: Easy Install Test steps: - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Perform production easy install run: | python3 ${GITHUB_WORKSPACE}/easy-install.py -p -n actions_test --email test@frappe.io From 934b2677c8e2fdfb05ff764e5ff85b18ea28e491 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 3 May 2023 13:52:01 +0530 Subject: [PATCH 42/54] chore: fix typo (#1448) --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 72f4825af..797ec4008 100755 --- a/bench/app.py +++ b/bench/app.py @@ -315,7 +315,7 @@ def get_excluded_apps(bench_path="."): def add_to_excluded_apps_txt(app, bench_path="."): if app == "frappe": - raise ValueError("Frappe app cannot be excludeed from update") + raise ValueError("Frappe app cannot be excluded from update") if app not in os.listdir("apps"): raise ValueError(f"The app {app} does not exist") apps = get_excluded_apps(bench_path=bench_path) From f3b55f7785710cf293aaa00a087b1bd7e760e7cf Mon Sep 17 00:00:00 2001 From: Alok Singh Date: Thu, 4 May 2023 17:28:34 +0530 Subject: [PATCH 43/54] fix: resolve filesystem app dependencies (#1450) * fix: resolve private app dependencies * fix: removed additional check and moved self.on_disk to previous check. --- bench/app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bench/app.py b/bench/app.py index 797ec4008..25724b5d9 100755 --- a/bench/app.py +++ b/bench/app.py @@ -81,17 +81,15 @@ def setup_details(self): if not self.is_repo: self.repo = self.app_name = self.name return + # fetch meta from installed apps if self.bench and os.path.exists(os.path.join(self.bench.name, "apps", self.name)): self.mount_path = os.path.join(self.bench.name, "apps", self.name) self.from_apps = True - self._setup_details_from_mounted_disk() - - # fetch meta for repo on mounted disk - elif os.path.exists(self.mount_path): self.on_disk = True self._setup_details_from_mounted_disk() + # fetch meta for repo from remote git server - traditional get-app url elif is_git_url(self.name): self.is_url = True From 082692cae2ceb26564e54d744531ac90c771188b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 9 May 2023 21:31:33 +0530 Subject: [PATCH 44/54] Revert "fix: resolve filesystem app dependencies (#1450)" (#1452) This reverts commit f3b55f7785710cf293aaa00a087b1bd7e760e7cf. --- bench/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 25724b5d9..797ec4008 100755 --- a/bench/app.py +++ b/bench/app.py @@ -81,14 +81,16 @@ def setup_details(self): if not self.is_repo: self.repo = self.app_name = self.name return - # fetch meta from installed apps if self.bench and os.path.exists(os.path.join(self.bench.name, "apps", self.name)): self.mount_path = os.path.join(self.bench.name, "apps", self.name) self.from_apps = True - self.on_disk = True self._setup_details_from_mounted_disk() + # fetch meta for repo on mounted disk + elif os.path.exists(self.mount_path): + self.on_disk = True + self._setup_details_from_mounted_disk() # fetch meta for repo from remote git server - traditional get-app url elif is_git_url(self.name): From a1d2c5226b598ed4cfbb0a67b943bb6ab1ae825c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 21 Apr 2023 15:19:04 +0530 Subject: [PATCH 45/54] fix: Print failing command with CommandFailedError I've seen way too many issues being faced / misunderstandings due to the lack of "cmd" --- bench/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index b9e0cced4..3fe17ad2e 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -155,7 +155,7 @@ def exec_cmd(cmd, cwd=".", env=None, _raise=True): if return_code: logger.warning(f"{cmd_log} executed with exit code {return_code}") if _raise: - raise CommandFailedError from subprocess.CalledProcessError(return_code, cmd) + raise CommandFailedError(cmd) from subprocess.CalledProcessError(return_code, cmd) return return_code From 817523943414345bdd161b50c263f634b7a8f70d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 21 Apr 2023 15:48:33 +0530 Subject: [PATCH 46/54] fix: Hide irrelevant stacks during exc raise --- bench/cli.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bench/cli.py b/bench/cli.py index 7f262407d..8ced2ef17 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -118,6 +118,8 @@ def cli(): _opts = [x.opts + x.secondary_opts for x in bench_command.params] opts = {item for sublist in _opts for item in sublist} + setup_exception_handler() + # handle usages like `--use-feature='feat-x'` and `--use-feature 'feat-x'` if cmd_from_sys and cmd_from_sys.split("=", 1)[0].strip() in opts: bench_command() @@ -240,3 +242,17 @@ def _chdir(*args, **kwargs): return f(*args, **kwargs) os.chdir = _chdir + + +def setup_exception_handler(): + from traceback import format_exception + + def handle_exception(exc_type, exc_info, tb): + print("".join(generate_exc(exc_type, exc_info, tb))) + + def generate_exc(exc_type, exc_info, tb): + for t in format_exception(exc_type, exc_info, tb): + if "/click/" not in t: + yield t + + sys.excepthook = handle_exception From 94a1b8c3fb8bee513c02841f48a2d05ccaab6a53 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 May 2023 15:40:11 +0530 Subject: [PATCH 47/54] fix: Setup exception handler for CommandFailedError only * Match only certain module lib paths --- bench/cli.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 8ced2ef17..215d47cd1 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -28,6 +28,8 @@ get_cmd_from_sysargv, ) from bench.utils.bench import get_env_cmd +from importlib.util import find_spec + # these variables are used to show dynamic outputs on the terminal dynamic_feed = False @@ -38,6 +40,7 @@ change_uid_msg = "You should not run this command as root" src = os.path.dirname(__file__) +SKIP_MODULE_TRACEBACK = ("click",) @contextmanager @@ -246,13 +249,22 @@ def _chdir(*args, **kwargs): def setup_exception_handler(): from traceback import format_exception + from bench.exceptions import CommandFailedError def handle_exception(exc_type, exc_info, tb): - print("".join(generate_exc(exc_type, exc_info, tb))) + if exc_type == CommandFailedError: + print("".join(generate_exc(exc_type, exc_info, tb))) + else: + sys.__excepthook__(exc_type, exc_info, tb) def generate_exc(exc_type, exc_info, tb): - for t in format_exception(exc_type, exc_info, tb): - if "/click/" not in t: - yield t + TB_SKIP = [ + os.path.dirname(find_spec(module).origin) for module in SKIP_MODULE_TRACEBACK + ] + + for tb_line in format_exception(exc_type, exc_info, tb): + for skip_module in TB_SKIP: + if skip_module not in tb_line: + yield tb_line sys.excepthook = handle_exception From 3af6058d1bb41025a65b423af028657fe88500f0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 21 Apr 2023 15:05:16 +0530 Subject: [PATCH 48/54] fix: Archive app for different repo/app names --- bench/app.py | 4 ++-- bench/bench.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 797ec4008..951352784 100755 --- a/bench/app.py +++ b/bench/app.py @@ -198,7 +198,7 @@ def get(self): @step(title="Archiving App {repo}", success="App {repo} Archived") def remove(self, no_backup: bool = False): - active_app_path = os.path.join("apps", self.repo) + active_app_path = os.path.join("apps", self.app_name) if no_backup: if not os.path.islink(active_app_path): @@ -209,7 +209,7 @@ def remove(self, no_backup: bool = False): else: archived_path = os.path.join("archived", "apps") archived_name = get_available_folder_name( - f"{self.repo}-{date.today()}", archived_path + f"{self.app_name}-{date.today()}", archived_path ) archived_app_path = os.path.join(archived_path, archived_name) diff --git a/bench/bench.py b/bench/bench.py index cdba90180..c2c70c37c 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -131,6 +131,9 @@ def uninstall(self, app, no_backup=False, force=False): except InvalidRemoteException: if not force: raise + except ValueError: + pass + self.apps.sync() # self.build() - removed because it seems unnecessary self.reload(_raise=False) From 4f423923b4c0aa646e9c4f1d7a04bf10bc420f61 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 11 May 2023 12:55:19 +0530 Subject: [PATCH 49/54] fix: Use App.app_name over repo attr --- bench/app.py | 14 +++++++------- bench/bench.py | 4 ++-- bench/tests/test_init.py | 4 ++-- bench/tests/test_utils.py | 4 +++- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/bench/app.py b/bench/app.py index 951352784..b95718b53 100755 --- a/bench/app.py +++ b/bench/app.py @@ -233,7 +233,7 @@ def install( verbose = bench.cli.verbose or verbose app_name = get_app_name(self.bench.name, self.app_name) - if not resolved and self.repo != "frappe" and not ignore_resolution: + if not resolved and self.app_name != "frappe" and not ignore_resolution: click.secho( f"Ignoring dependencies of {self.name}. To install dependencies use --resolve-deps", fg="yellow", @@ -262,13 +262,13 @@ def _get_dependencies(self): from bench.utils.app import get_required_deps, required_apps_from_hooks if self.on_disk: - required_deps = os.path.join(self.mount_path, self.repo, "hooks.py") + required_deps = os.path.join(self.mount_path, self.app_name, "hooks.py") try: return required_apps_from_hooks(required_deps, local=True) except IndexError: return [] try: - required_deps = get_required_deps(self.org, self.repo, self.tag or self.branch) + required_deps = get_required_deps(self.org, self.app_name, self.tag or self.branch) return required_apps_from_hooks(required_deps) except Exception: return [] @@ -290,16 +290,16 @@ def make_resolution_plan(app: App, bench: "Bench"): decide what apps and versions to install and in what order """ resolution = OrderedDict() - resolution[app.repo] = app + resolution[app.app_name] = app for app_name in app._get_dependencies(): dep_app = App(app_name, bench=bench) is_valid_frappe_branch(dep_app.url, dep_app.branch) dep_app.required_by = app.name - if dep_app.repo in resolution: - click.secho(f"{dep_app.repo} is already resolved skipping", fg="yellow") + if dep_app.app_name in resolution: + click.secho(f"{dep_app.app_name} is already resolved skipping", fg="yellow") continue - resolution[dep_app.repo] = dep_app + resolution[dep_app.app_name] = dep_app resolution.update(make_resolution_plan(dep_app, bench)) app.local_resolution = [repo_name for repo_name, _ in reversed(resolution.items())] return resolution diff --git a/bench/bench.py b/bench/bench.py index c2c70c37c..0f0e27d0a 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -312,13 +312,13 @@ def insert(self, key, value): def add(self, app: "App"): app.get() app.install() - super().append(app.repo) + super().append(app.app_name) self.apps.sort() def remove(self, app: "App", no_backup: bool = False): app.uninstall() app.remove(no_backup=no_backup) - super().remove(app.repo) + super().remove(app.app_name) def append(self, app: "App"): return self.add(app) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index b0a871a4a..56634e67a 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -28,8 +28,8 @@ def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) app = App("file:///tmp/frappe") self.assertTupleEqual( - (app.mount_path, app.url, app.repo, app.org), - ("/tmp/frappe", "file:///tmp/frappe", "frappe", "frappe"), + (app.mount_path, app.url, app.repo, app.app_name, app.org), + ("/tmp/frappe", "file:///tmp/frappe", "frappe", "frappe", "frappe"), ) self.assert_folders(bench_name) self.assert_virtual_env(bench_name) diff --git a/bench/tests/test_utils.py b/bench/tests/test_utils.py index e0137dcaf..2f645497c 100644 --- a/bench/tests/test_utils.py +++ b/bench/tests/test_utils.py @@ -101,4 +101,6 @@ def test_app_states(self): def test_ssh_ports(self): app = App("git@github.com:22:frappe/frappe") - self.assertEqual((app.use_ssh, app.org, app.repo), (True, "frappe", "frappe")) + self.assertEqual( + (app.use_ssh, app.org, app.repo, app.app_name), (True, "frappe", "frappe", "frappe") + ) From e80daf8c3e78ae5a675d5c9d1b75a92d1fe57eae Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 31 May 2023 13:57:27 +0530 Subject: [PATCH 50/54] fix: Use == for dict comparison --- bench/utils/bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 3fe7f85b6..16fc206e3 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -155,7 +155,7 @@ def update_npm_packages(bench_path=".", apps=None): else: package_json[key] = value - if package_json is {}: + if package_json == {}: with open(os.path.join(os.path.dirname(__file__), "package.json")) as f: package_json = json.loads(f.read()) From f7fbee99d61a9fdf727a3efe1419eceb4148fb35 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 31 May 2023 13:59:34 +0530 Subject: [PATCH 51/54] fix: Revert usage of app_name for repo name --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index b95718b53..48fbb985b 100755 --- a/bench/app.py +++ b/bench/app.py @@ -268,7 +268,7 @@ def _get_dependencies(self): except IndexError: return [] try: - required_deps = get_required_deps(self.org, self.app_name, self.tag or self.branch) + required_deps = get_required_deps(self.org, self.repo, self.tag or self.branch) return required_apps_from_hooks(required_deps) except Exception: return [] From d9c8335fbf3344c982b604ffd1507c4c0cd4ff11 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 2 Jun 2023 12:41:56 +0530 Subject: [PATCH 52/54] fix: Remove except-pass ValueError --- bench/bench.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 0f0e27d0a..c21e34407 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -131,8 +131,6 @@ def uninstall(self, app, no_backup=False, force=False): except InvalidRemoteException: if not force: raise - except ValueError: - pass self.apps.sync() # self.build() - removed because it seems unnecessary From 036cb6291ec2d6f837dc1db40738f688034da5b0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 4 Jul 2023 19:52:05 +0530 Subject: [PATCH 53/54] fix: don't compile python files manually. (#1469) --- bench/commands/update.py | 2 +- bench/utils/bench.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index d03c0f86e..63b776527 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -33,7 +33,7 @@ @click.option( "--no-compile", is_flag=True, - help="If set, Python bytecode won't be compiled before restarting the processes", + help="[DEPRECATED] This flag doesn't do anything now.", ) @click.option("--force", is_flag=True, help="Forces major version upgrades") @click.option( diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 16fc206e3..1fede3a22 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -456,13 +456,6 @@ def update( if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - if pull and compile: - from compileall import compile_dir - - print("Compiling Python files...") - apps_dir = os.path.join(bench_path, "apps") - compile_dir(apps_dir, quiet=1, rx=re.compile(".*node_modules.*")) - bench.reload(web=False, supervisor=restart_supervisor, systemd=restart_systemd) conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) From 77ebdbe6f93e98a46a8926cc2f8a9bac6b83d3f7 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 7 Aug 2023 12:27:02 +0530 Subject: [PATCH 54/54] ci: fix test versions (#1474) --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5c442600..74b1b0d21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,9 +114,17 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - uses: actions/setup-node@v3 + if: ${{ matrix.python-version == '3.10' }} + with: + node-version: 18 + + - uses: actions/setup-node@v3 + if: ${{ matrix.python-version == '3.7' }} with: node-version: 14 + - run: | wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb; sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;