-
Notifications
You must be signed in to change notification settings - Fork 113
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #201 from mendix/mxbuild-externally
Run MxBuild outside CF Buildpack
- Loading branch information
Showing
18 changed files
with
465 additions
and
135 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,250 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import argparse | ||
| import pathlib | ||
| import os | ||
| import tempfile | ||
| import json | ||
| import sqlite3 | ||
| import zipfile | ||
| import atexit | ||
| import shutil | ||
| import subprocess | ||
| import sys | ||
| import selectors | ||
| import logging | ||
| import platform | ||
|
|
||
| logging.basicConfig( | ||
| level=logging.INFO, | ||
| stream=sys.stdout | ||
| ) | ||
|
|
||
| def find_default_file(source, ext): | ||
| if os.path.isfile(source): | ||
| return source if source.name.endswith(ext) else None | ||
| files = [x for x in os.listdir(source) if x.endswith(ext)] | ||
| if len(files) == 1: | ||
| return os.path.join(source, files[0]) | ||
| if len(files) > 1: | ||
| raise Exception(f"More than one {ext} file found, can not continue") | ||
| return None | ||
|
|
||
| def get_metadata_value(source_dir): | ||
| file_name = os.path.join(source_dir, 'model', 'metadata.json') | ||
| try: | ||
| with open(file_name) as file_handle: | ||
| return json.loads(file_handle.read()) | ||
| except IOError: | ||
| return None | ||
|
|
||
| def extract_zip(mda_file): | ||
| temp_dir = tempfile.TemporaryDirectory(prefix='mendix-docker-buildpack') | ||
| with zipfile.ZipFile(mda_file) as zip_file: | ||
| zip_file.extractall(temp_dir.name) | ||
| return temp_dir | ||
|
|
||
| BUILDER_PROCESS = None | ||
| def stop_processes(): | ||
| if BUILDER_PROCESS is not None: | ||
| proc = BUILDER_PROCESS | ||
| proc.terminate() | ||
| proc.communicate() | ||
| proc.wait() | ||
|
|
||
| def container_call(args): | ||
| build_executables = ['podman', 'docker'] | ||
| build_executable = None | ||
| logger_stdout = None | ||
| logger_stderr = None | ||
| for builder in build_executables: | ||
| build_executable = shutil.which(builder) | ||
| if build_executable is not None: | ||
| logger_stderr = logging.getLogger(builder + '-stderr') | ||
| logger_stdout = logging.getLogger(builder + '-stdout') | ||
| break | ||
| if build_executable is None: | ||
| raise Exception('Cannot find Podman or Docker executable') | ||
| proc = subprocess.Popen([build_executable] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) | ||
| BUILDER_PROCESS = proc | ||
|
|
||
| sel = selectors.DefaultSelector() | ||
| sel.register(proc.stdout, selectors.EVENT_READ) | ||
| sel.register(proc.stderr, selectors.EVENT_READ) | ||
|
|
||
| last_line_stdout = None | ||
| last_line_stderr = None | ||
| stdout_open, stderr_open = True, True | ||
| while stdout_open or stderr_open: | ||
| for key, _ in sel.select(): | ||
| data = key.fileobj.readline() | ||
| if data == '': | ||
| if key.fileobj is proc.stdout: | ||
| stdout_open = False | ||
| elif key.fileobj is proc.stderr: | ||
| stderr_open = False | ||
| continue | ||
| data = data.rstrip() | ||
| if key.fileobj is proc.stdout: | ||
| last_line_stdout = data | ||
| logger_stdout.info(data) | ||
| elif key.fileobj is proc.stderr: | ||
| last_line_stderr = data | ||
| # stderr is mostly used for progress notifications, not errors | ||
| logger_stderr.info(data) | ||
|
|
||
| sel.close() | ||
| BUILDER_PROCESS = None | ||
| if proc.wait() != 0: | ||
| raise Exception(f"Builder returned with error: {last_line_stderr}") | ||
| return last_line_stdout | ||
|
|
||
| def pull_image(image_url): | ||
| try: | ||
| container_call(['image', 'pull', image_url]) | ||
| return image_url | ||
| except: | ||
| return None | ||
|
|
||
| def delete_container(container_id): | ||
| try: | ||
| container_call(['container', 'rm', '--force', container_id]) | ||
| except Exception as e: | ||
| logging.warning('Failed to delete container {}: {}'.format(container_id, e)) | ||
|
|
||
| def build_mpr_builder(mx_version, dotnet, artifacts_repository=None): | ||
| builder_image_tag = f"mxbuild-{mx_version}-{dotnet}-{platform.machine()}" | ||
| builder_image_url = None | ||
| if artifacts_repository is not None: | ||
| builder_image_url = f"{artifacts_repository}:{builder_image_tag}" | ||
| image_url = pull_image(builder_image_url) | ||
| if image_url is not None: | ||
| return image_url | ||
| else: | ||
| builder_image_url = f"mendix-buildpack:{builder_image_tag}" | ||
|
|
||
| prefix = '' | ||
| if platform.machine() == 'arm64' and dotnet == 'dotnet': | ||
| prefix = 'arm64-' | ||
|
|
||
| mxbuild_filename = f"{prefix}mxbuild-{mx_version}.tar.gz" | ||
| mxbuild_url = f"https://download.mendix.com/runtimes/{mxbuild_filename}" | ||
|
|
||
| build_args = ['--build-arg', f"MXBUILD_DOWNLOAD_URL={mxbuild_url}", | ||
| '--file', os.path.join('mxbuild', f"{dotnet}.dockerfile"), | ||
| '--tag', builder_image_url] | ||
|
|
||
| container_call(['image', 'build'] + build_args + ['mxbuild']) | ||
| if artifacts_repository is not None: | ||
| try: | ||
| container_call(['image', 'push', builder_image_url]) | ||
| except Exception as e: | ||
| logging.warning('Failed to push mxbuild into artifacts repository: {}; continuing with the build'.format(e)) | ||
| return builder_image_url | ||
|
|
||
| def get_git_commit(source_dir): | ||
| git_head = os.path.join(source_dir, '.git', 'HEAD') | ||
| if not os.path.isfile(git_head): | ||
| raise Exception('Project source doesn\'t contain git metadata') | ||
| with open(git_head) as git_head: | ||
| git_head_line = git_head.readline().split() | ||
| if len(git_head_line) == 1: | ||
| # Detached commit | ||
| return git_head_line[0] | ||
| if len(git_head_line) > 2: | ||
| raise Exception(f"Unsupported Git HEAD format {git_head_line}") | ||
| git_branch = git_head_line[1].split('/') | ||
| git_branch_file = os.path.join(*([source_dir, '.git'] + git_branch)) | ||
| if not os.path.isfile(git_branch_file): | ||
| raise Exception('Git branch file doesn\'t exist') | ||
| with open(git_branch_file) as git_branch_file: | ||
| return git_branch_file.readline() | ||
|
|
||
|
|
||
| def build_mpr(source_dir, mpr_file, destination, artifacts_repository=None): | ||
| cursor = sqlite3.connect(mpr_file).cursor() | ||
| cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1") | ||
| mx_version = cursor.fetchone()[0] | ||
| mx_version_value = parse_version(mx_version) | ||
| logging.debug('Detected Mendix version {}'.format('.'.join(map(str,mx_version_value)))) | ||
| dotnet = 'dotnet' if mx_version_value >= (10, 0, 0, 0) else 'mono' | ||
| builder_image = build_mpr_builder(mx_version, dotnet, artifacts_repository) | ||
| model_version = None | ||
| try: | ||
| model_version = get_git_commit(source_dir) | ||
| except Exception as e: | ||
| model_version = 'unversioned' | ||
| logging.warning('Cannot determine git commit ({}), will set model version to unversioned'.format(e)) | ||
| container_id = container_call(['container', 'create', builder_image, os.path.basename(mpr_file), model_version]) | ||
| atexit.register(delete_container, container_id) | ||
| container_call(['container', 'cp', os.path.abspath(source_dir)+'/.', f"{container_id}:/workdir/project"]) | ||
| build_result = container_call(['start', '--attach', '--interactive', container_id]) | ||
|
|
||
| temp_dir = tempfile.TemporaryDirectory(prefix='mendix-docker-buildpack') | ||
| container_call(['container', 'cp', f"{container_id}:/workdir/output.mda", temp_dir.name]) | ||
| with zipfile.ZipFile(os.path.join(temp_dir.name, 'output.mda')) as zip_file: | ||
| zip_file.extractall(destination) | ||
|
|
||
| def parse_version(version): | ||
| return tuple([ int(n) for n in version.split('.') ]) | ||
|
|
||
| def prepare_destination(destination_path): | ||
| with os.scandir(destination_path) as entries: | ||
| for entry in entries: | ||
| if entry.is_dir() and not entry.is_symlink(): | ||
| shutil.rmtree(entry.path) | ||
| else: | ||
| os.remove(entry.path) | ||
| project_path = os.path.join(destination_path, 'project') | ||
| os.mkdir(project_path, 0o755) | ||
| shutil.copytree('scripts', os.path.join(destination_path, 'scripts')) | ||
| shutil.copyfile('Dockerfile', os.path.join(destination_path, 'Dockerfile')) | ||
| return project_path | ||
|
|
||
| def prepare_mda(source_path, destination_path, artifacts_repository=None): | ||
| destination_path = prepare_destination(destination_path) | ||
| mpk_file = find_default_file(source_path, '.mpk') | ||
| extracted_dir = None | ||
| if mpk_file is not None: | ||
| extracted_dir = extract_zip(mpk_file) | ||
| source_path = extracted_dir.name | ||
| mpr_file = find_default_file(source_path, '.mpr') | ||
| if mpr_file is not None: | ||
| source_path = os.path.abspath(os.path.join(mpr_file, os.pardir)) | ||
| return build_mpr(source_path, mpr_file, destination_path, artifacts_repository) | ||
| mda_file = find_default_file(source_path, '.mda') | ||
| if mda_file is not None: | ||
| with zipfile.ZipFile(mda_file) as zip_file: | ||
| zip_file.extractall(destination_path) | ||
| elif os.path.isdir(source_path): | ||
| shutil.copytree(source_path, destination_path, dirs_exist_ok=True) | ||
| extracted_mda_file = get_metadata_value(destination_path) | ||
| if extracted_mda_file is not None: | ||
| return destination_path | ||
| else: | ||
| raise Exception('No supported files found in source path') | ||
|
|
||
| def build_image(mda_dir): | ||
| # TODO: build the full image, or just copy MDA into destination? | ||
| mda_path = mda_dir.name if isinstance(mda_dir, tempfile.TemporaryDirectory) else mda_dir | ||
| mda_metadata = get_metadata_value(mda_path) | ||
| mx_version = mda_metadata['RuntimeVersion'] | ||
| java_version = mda_metadata.get('JavaVersion', 11) | ||
| logging.debug("Detected Mendix {} Java {}".format(mx_version, java_version)) | ||
|
|
||
| if __name__ == '__main__': | ||
| parser = argparse.ArgumentParser(description='Build a Mendix app') | ||
| parser.add_argument('--source', metavar='source', required=True, nargs='?', type=pathlib.Path, help='Path to source Mendix app (MDA file, MPK file, MPR directory or extracted MDA directory)') | ||
| parser.add_argument('--destination', metavar='destination', required=True, nargs='?', type=pathlib.Path, help='Destination for MDA') | ||
| parser.add_argument('--artifacts-repository', required=False, nargs='?', metavar='artifacts_repository', type=str, help='Repository to use for caching build images') | ||
| parser.add_argument('action', metavar='action', choices=['build-mda-dir'], help='Action to perform') | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| atexit.register(stop_processes) | ||
| try: | ||
| prepare_mda(args.source, args.destination, args.artifacts_repository) | ||
| except KeyboardInterrupt: | ||
| stop_processes() | ||
| raise | ||
| # build_image(args.destination) |
Oops, something went wrong.