diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f178af8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +sudo: false +cache: pip + +language: python + +python: + - "3.6" + +before_install: + # For tests running git command + - git config --global user.name "test" + - git config --global user.email "test.test@test.com" + +# command to install dependencies +install: + - pip install -r requirements.txt + - pip install -r test_requirements.txt + +# command to run tests +script: + - git --version + - flake8 . --exclude=__init__.py + - coverage run --source git_aggregator setup.py test + +after_success: + - coveralls diff --git a/DEVELOP.rst b/DEVELOP.rst index 2342862..f6309f0 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -110,6 +110,13 @@ How to improve the library * Read (or complete !) the migration advices of the OCA. https://github.com/OCA/maintainer-tools/wiki#migration +* Read the complementary pages + https://odoo-development.readthedocs.io/en/latest/migration/ + +* Discover what changed between two revisions, reading OpenUpgrade + documentation, specially the modules changes, for exemple: + https://github.com/OCA/OpenUpgrade/blob/12.0/odoo/openupgrade/doc/source/modules110-120.rst + * Create or complete the according migration file. * Add tests. diff --git a/README.rst b/README.rst index c5c5915..db6a836 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,10 @@ :alt: License: AGPL-3 .. image:: https://img.shields.io/badge/python-3.6-blue.svg :alt: Python support: 3.6 +.. image:: https://travis-ci.org/grap/odoo-migrate.svg?branch=master + :target: https://travis-ci.org/grap/odoo-migrate +.. image:: https://coveralls.io/repos/grap/odoo-migrate/badge.png?branch=master + :target: https://coveralls.io/r/grap/odoo-migrate?branch=master ============ odoo-migrate @@ -120,6 +124,9 @@ Available arguments |``--log-level`` | ``-ll`` | Default: | Possible value: ``DEBUG``, ``INFO``, ``WARNING``, etc.| | | | ``INFO`` | | +--------------------------+----------+-----------------+-------------------------------------------------------+ +|``--no-commit`` | ``-nc`` | Default: | If set the library will not git add and git commit | +| | | commit | changes. | ++--------------------------+----------+-----------------+-------------------------------------------------------+ Roadmap / Know issues @@ -129,6 +136,14 @@ Roadmap / Know issues * Add tests. +Changes +======= + +0.1.2 (October 10, 2019) +------------------------ + +* First release + Credits ======= @@ -141,3 +156,4 @@ Contributors ------------ * Sylvain LE GAL (https://www.twitter.com/legalsylvain) + diff --git a/odoo_migrate/__main__.py b/odoo_migrate/__main__.py index cc3f291..40f1ef9 100644 --- a/odoo_migrate/__main__.py +++ b/odoo_migrate/__main__.py @@ -92,6 +92,15 @@ def get_parser(): type=str, ) + main_parser.add_argument( + "-nc", + "--no-commit", + action='store_true', + default=False, + help="Enable this option, if you don't want that the library commits" + " the changes. (using git add and git commit command)" + ) + return main_parser @@ -111,73 +120,13 @@ def main(): migration = Migration( args.directory, args.init_version_name, args.target_version_name, - module_names, args.format_patch, args.remote_name, args.force_black + module_names, args.format_patch, args.remote_name, + args.force_black, not args.no_commit, ) # run Migration migration.run() - # # Get Main path and test if exists - # root_path = Path(args.directory) - # if not root_path.exists(): - # raise ValueError("'%s' is not a valid path." % (args.directory)) - - # # Recover modules list - # modules_path = [] - # modules_list = args.modules\ - # and [x for x in args.modules.split(",") if x] or [] - - # if not args.modules: - # # Recover all submodules, if no modules list is provided - # subfolders = [x for x in root_path.iterdir() if x.is_dir()] - # else: - # subfolders = [root_path / x for x in modules_list] - - # # Recover code for former version, if asked - # if args.format_patch: - # if len(modules_list) != 1: - # raise ValueError( - # "If 'format-patch' option is enabled, you should provide" - # " a unique module name in the 'modules' argument.") - # migrate_tools._get_code_from_previous_branch( - # _logger, root_path, modules_list[0], args.init_version, - # args.target_version, args.remote_name) - - # # check if each folder is a valid module or not - # for subfolder in subfolders: - # if (subfolder / "__openerp__.py").exists() or ( - # subfolder / "__manifest__.py" - # ).exists(): - # modules_path.append(subfolder) - # else: - # if modules_list: - # _logger.warning( - # "The module %s was not found in the directory %s" - # % (subfolder.name, args.directory) - # ) - - # if modules_path: - # _logger.debug( - # "The lib will process the following modules '%s'" - # % (", ".join([x.name for x in modules_path])) - # ) - # else: - # _logger.error("No module found.") - - # # Compute migration list - # migration_list = tools._get_migration_list( - # args.init_version, args.target_version - # ) - - # for module_path in modules_path: - # # Use black to clean the code - # if args.enable_black: - # pass - - # # migrate modules - # migrate_tools._migrate_module( - # _logger, root_path, module_path, migration_list) - except KeyboardInterrupt: pass diff --git a/odoo_migrate/config.py b/odoo_migrate/config.py index 0d3278c..ed2520c 100644 --- a/odoo_migrate/config.py +++ b/odoo_migrate/config.py @@ -42,3 +42,5 @@ _ALLOWED_EXTENSIONS = [".py", ".xml", ".js"] _BLACK_LINE_LENGTH = 79 + +_MANIFEST_NAMES = ["__openerp__.py", "__manifest__.py"] diff --git a/odoo_migrate/migration.py b/odoo_migrate/migration.py index 4f07ea4..c07b220 100644 --- a/odoo_migrate/migration.py +++ b/odoo_migrate/migration.py @@ -7,7 +7,7 @@ import pathlib import pkgutil -from .config import _AVAILABLE_MIGRATION_STEPS +from .config import _AVAILABLE_MIGRATION_STEPS, _MANIFEST_NAMES from .exception import ConfigException from .log import logger from .tools import _execute_shell, _get_latest_version_code @@ -21,16 +21,18 @@ class Migration(): _use_black = False _migration_scripts = [] _module_migrations = [] + _commit_enabled = True def __init__( self, relative_directory_path, init_version_name, target_version_name, module_names=[], format_patch=False, remote_name='origin', - force_black=False + force_black=False, commit_enabled=True, ): - pass + self._use_black = force_black + self._commit_enabled = commit_enabled + # Get migration steps that will be runned found = False - self._use_black = force_black for item in _AVAILABLE_MIGRATION_STEPS: if not found and item["init_version_name"] != init_version_name: continue @@ -94,8 +96,7 @@ def __init__( self._get_migration_scripts() def _is_module_path(self, module_path): - return (module_path / "__openerp__.py").exists() or\ - (module_path / "__manifest__.py").exists() + return any([(module_path / x).exists() for x in _MANIFEST_NAMES]) def _get_code_from_previous_branch(self, module_name, remote_name): init_version = self._migration_steps[0]["init_version_name"] @@ -170,7 +171,7 @@ def _get_migration_scripts(self): [x.__file__.split('/')[-1] for x in self._migration_scripts])) def run(self): - logger.info( + logger.debug( "Running migration from: %s to: %s in '%s'" % ( self._migration_steps[0]["init_version_name"], self._migration_steps[-1]["target_version_name"], diff --git a/odoo_migrate/migration_scripts/migrate_080_090.py b/odoo_migrate/migration_scripts/migrate_080_090.py index d12f34c..3777b6e 100644 --- a/odoo_migrate/migration_scripts/migrate_080_090.py +++ b/odoo_migrate/migration_scripts/migrate_080_090.py @@ -1,3 +1,7 @@ +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + _TEXT_REPLACES = { "*": { "base.group_configuration": "base.group_system", diff --git a/odoo_migrate/migration_scripts/migrate_080_allways.py b/odoo_migrate/migration_scripts/migrate_080_allways.py index e697060..55d83c4 100644 --- a/odoo_migrate/migration_scripts/migrate_080_allways.py +++ b/odoo_migrate/migration_scripts/migrate_080_allways.py @@ -1,3 +1,7 @@ +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + _FILE_RENAMES = {"__openerp__.py": "__manifest__.py"} _TEXT_REPLACES = { diff --git a/odoo_migrate/migration_scripts/migrate_100_110.py b/odoo_migrate/migration_scripts/migrate_100_110.py index 509be48..aaa1602 100644 --- a/odoo_migrate/migration_scripts/migrate_100_110.py +++ b/odoo_migrate/migration_scripts/migrate_100_110.py @@ -1,2 +1,7 @@ -_TEXT_WARNING = {"*": {"ir.values": "'ir.values' found." - " This model has been removed in V11."}} +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +_TEXT_WARNING = {"*": { + "ir.values": "[V11] Reference to 'ir.values'." + " This model has been removed."}} diff --git a/odoo_migrate/migration_scripts/migrate_100_allways.py b/odoo_migrate/migration_scripts/migrate_100_allways.py index c92588c..7de2a2c 100644 --- a/odoo_migrate/migration_scripts/migrate_100_allways.py +++ b/odoo_migrate/migration_scripts/migrate_100_allways.py @@ -1 +1,5 @@ +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + _TEXT_REPLACES = {".py": {r"#.+coding:\Wutf-8.+\n": ""}} diff --git a/odoo_migrate/migration_scripts/migrate_110_120.py b/odoo_migrate/migration_scripts/migrate_110_120.py index 0959edd..37e30f8 100644 --- a/odoo_migrate/migration_scripts/migrate_110_120.py +++ b/odoo_migrate/migration_scripts/migrate_110_120.py @@ -1,3 +1,7 @@ +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + _TEXT_REPLACES = { ".py": { "from odoo.addons.base.res": "from odoo.addons.base.models", diff --git a/odoo_migrate/migration_scripts/migrate_120_130.py b/odoo_migrate/migration_scripts/migrate_120_130.py new file mode 100644 index 0000000..7a13cfa --- /dev/null +++ b/odoo_migrate/migration_scripts/migrate_120_130.py @@ -0,0 +1,7 @@ +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +_TEXT_WARNING = {"*": { + "web_settings_dashboard": "[V13] Reference to 'web_settings_dashboard'" + ". This module has been removed."}} diff --git a/odoo_migrate/migration_scripts/migrate_allways.py b/odoo_migrate/migration_scripts/migrate_allways.py index 325a559..0279975 100644 --- a/odoo_migrate/migration_scripts/migrate_allways.py +++ b/odoo_migrate/migration_scripts/migrate_allways.py @@ -1,9 +1,34 @@ -# Bump Module version -# Switch the installable key to True +# Copyright (C) 2019 - Today: GRAP (http://www.grap.coop) +# @author: Sylvain LE GAL (https://twitter.com/legalsylvain) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + import os import subprocess +def bump_revision(**kwargs): + tools = kwargs['tools'] + manifest_path = kwargs['manifest_path'] + target_version_name = kwargs['migration_steps'][-1]["target_version_name"] + + new_version = "%s.1.0.0" % target_version_name + + old_term = r"('|\")version('|\").*('|\").*('|\")" + new_term = r'\1version\2: "{0}"'.format(new_version) + tools._replace_in_file( + manifest_path, {old_term: new_term}, + "Bump version to %s" % new_version) + + +def set_module_installable(**kwargs): + tools = kwargs['tools'] + manifest_path = kwargs['manifest_path'] + old_term = r"('|\")installable('|\").*(False)" + new_term = r"\1installable\2: True" + tools._replace_in_file( + manifest_path, {old_term: new_term}, "Set module installable") + + def remove_migration_folder(**kwargs): logger = kwargs['logger'] module_path = kwargs['module_path'] @@ -14,4 +39,6 @@ def remove_migration_folder(**kwargs): module_name)) subprocess.check_output("rm -r %s" % migration_path_folder, shell=True) -_GLOBAL_FUNCTIONS = [remove_migration_folder] + +_GLOBAL_FUNCTIONS = [ + remove_migration_folder, set_module_installable, bump_revision] diff --git a/odoo_migrate/module_migration.py b/odoo_migrate/module_migration.py index 4195f19..f09ec8b 100644 --- a/odoo_migrate/module_migration.py +++ b/odoo_migrate/module_migration.py @@ -8,8 +8,9 @@ import re from .log import logger -from .config import _ALLOWED_EXTENSIONS, _BLACK_LINE_LENGTH +from .config import _ALLOWED_EXTENSIONS, _BLACK_LINE_LENGTH, _MANIFEST_NAMES from .tools import _execute_shell +from . import tools class ModuleMigration(): @@ -17,20 +18,38 @@ class ModuleMigration(): _migration = False _module_name = False _module_path = False + _manifest_path = False def __init__(self, migration, module_name): self._migration = migration self._module_name = module_name self._module_path = self._migration._directory_path / module_name + for manifest_name in _MANIFEST_NAMES: + manifest_path = self._module_path / manifest_name + if manifest_path.exists(): + self._manifest_path = manifest_path def run(self): - logger.info("Running migration of module %s" % self._module_name) + logger.info("[%s] Running migration from %s to %s" % ( + self._module_name, + self._migration._migration_steps[0]["init_version_name"], + self._migration._migration_steps[-1]["target_version_name"], + )) + + # Run Black, if required self._run_black() + self._commit_changes( + "[REF] %s: Black python code" % (self._module_name)) + + # Apply migration script for migration_script in self._migration._migration_scripts: self._run_migration_scripts(migration_script) + self._commit_changes("[MIG] %s: Migration to %s" % ( + self._module_name, + self._migration._migration_steps[-1]["target_version_name"])) + def _run_black(self): - has_change = False if not self._migration._use_black: return @@ -46,32 +65,16 @@ def _run_black(self): absolute_file_path = os.path.join(root, filename) - has_change = has_change or black.format_file_in_place( + black.format_file_in_place( pathlib.Path(absolute_file_path), False, file_mode, black.WriteBack.YES) - # Commit changes - if has_change: - commit_name = "[REF] %s: Black python code" % (self._module_name) - - logger.info( - "Commit changes for %s. commit name '%s'" % ( - self._module_name, commit_name - )) - - _execute_shell( - "cd %s && git add . --all && git commit -m '%s'" % ( - str(self._migration._directory_path.resolve()), commit_name - )) - def _run_migration_scripts(self, migration_script): file_renames = getattr(migration_script, "_FILE_RENAMES", {}) text_replaces = getattr(migration_script, "_TEXT_REPLACES", {}) text_warnings = getattr(migration_script, "_TEXT_WARNING", {}) global_functions = getattr(migration_script, "_GLOBAL_FUNCTIONS", {}) - has_change = False - for root, directories, filenames in os.walk( self._module_path.resolve()): for filename in filenames: @@ -87,37 +90,26 @@ def _run_migration_scripts(self, migration_script): # Rename file, if required new_name = file_renames.get(filename) if new_name: - has_change = True self._rename_file( self._migration._directory_path, absolute_file_path, os.path.join(root, new_name)) absolute_file_path = os.path.join(root, new_name) - with open(absolute_file_path, "U") as f: - # Operate changes in the file (replacements, removals) - current_text = f.read() - new_text = current_text - - replaces = text_replaces.get("*", {}) - replaces.update(text_replaces.get(extension, {})) + # Operate changes in the file (replacements, removals) + replaces = text_replaces.get("*", {}) + replaces.update(text_replaces.get(extension, {})) - for old_term, new_term in replaces.items(): - new_text = re.sub(old_term, new_term, new_text) + new_text = tools._replace_in_file( + absolute_file_path, replaces, "froma") - # Write file if changed - if new_text != current_text: - has_change = True - logger.info("Changing content of file: %s" % filename) - with open(absolute_file_path, "w") as f: - f.write(new_text) - - # Display warnings if the file contents some obsolete code - warnings = text_warnings.get("*", {}) - warnings.update(text_warnings.get(extension, {})) - for pattern, warning_message in warnings.items(): - if re.findall(pattern, new_text): - logger.warning(warning_message) + # Display warnings if the new content contains some obsolete + # pattern + warnings = text_warnings.get("*", {}) + warnings.update(text_warnings.get(extension, {})) + for pattern, warning_message in warnings.items(): + if re.findall(pattern, new_text): + logger.warning(warning_message) if global_functions: for function in global_functions: @@ -125,27 +117,15 @@ def _run_migration_scripts(self, migration_script): logger=logger, module_path=self._module_path, module_name=self._module_name, + manifest_path=self._manifest_path, + migration_steps=self._migration._migration_steps, + tools=tools, ) - # Commit changes - if has_change: - commit_name = "[MIG] %s: Migration to %s" % ( - self._module_name, - self._migration._migration_steps[-1]["target_version_name"]) - - logger.info( - "Commit changes for %s. commit name '%s'" % ( - self._module_name, commit_name - )) - - _execute_shell( - "cd %s && git add . --all && git commit -m '%s'" % ( - str(self._migration._directory_path.resolve()), commit_name - )) - def _rename_file(self, module_path, old_file_path, new_file_path): """ Rename a file. try to execute 'git mv', to avoid huge diff. + if 'git mv' fails, make a classical rename """ logger.info( @@ -158,3 +138,19 @@ def _rename_file(self, module_path, old_file_path, new_file_path): "cd %s && git mv %s %s" % ( str(module_path.resolve()), old_file_path, new_file_path )) + + def _commit_changes(self, commit_name): + if not self._migration._commit_enabled: + return + + if _execute_shell("cd %s && git diff" % ( + str(self._migration._directory_path.resolve()))): + logger.info( + "Commit changes for %s. commit name '%s'" % ( + self._module_name, commit_name + )) + + _execute_shell( + "cd %s && git add . --all && git commit -m '%s'" % ( + str(self._migration._directory_path.resolve()), commit_name + )) diff --git a/odoo_migrate/tools.py b/odoo_migrate/tools.py index 3bd7097..2f5f70b 100644 --- a/odoo_migrate/tools.py +++ b/odoo_migrate/tools.py @@ -3,6 +3,8 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import subprocess +import re + from .config import _AVAILABLE_MIGRATION_STEPS from .log import logger @@ -26,3 +28,21 @@ def _get_latest_version_code(): def _execute_shell(shell_command): logger.debug("Execute Shell:\n%s" % (shell_command)) return subprocess.check_output(shell_command, shell=True) + + +def _replace_in_file(file_path, replaces, log_message=False): + with open(file_path, "U") as f: + current_text = f.read() + new_text = current_text + + for old_term, new_term in replaces.items(): + new_text = re.sub(old_term, new_term, new_text) + + # Write file if changed + if new_text != current_text: + if not log_message: + log_message = "Changing content of file: %s" % file_path.name + logger.info(log_message) + with open(file_path, "w") as f: + f.write(new_text) + return new_text diff --git a/setup.py b/setup.py index b6347c6..ab298c2 100644 --- a/setup.py +++ b/setup.py @@ -6,14 +6,14 @@ setuptools.setup( name="odoo-migrate", - version="0.1.2", + version="0.1.3", author="GRAP, Groupement Régional Alimentaire de Proximité", author_email="informatique@grap.coop", description="Small tools to migrate Odoo modules from a version" " to another", long_description=open('README.rst').read(), long_description_content_type='text/x-rst', - url="https://www.grap.coop", + url="https://github.com/grap/odoo-migrate", packages=['odoo_migrate', 'odoo_migrate.migration_scripts'], classifiers=[ "Programming Language :: Python :: 3", diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..88166fb --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +coveralls +flake8