From 365e731318c4423cbe08bc22776377a09bad9470 Mon Sep 17 00:00:00 2001 From: KOLANICH Date: Sat, 9 Jun 2018 18:49:53 +0300 Subject: [PATCH] Initial commit --- .editorconfig | 12 ++ .github/.templateMarker | 1 + .github/dependabot.yml | 8 + .github/workflows/CI.yml | 15 ++ .gitignore | 10 ++ .gitlab-ci.yml | 51 ++++++ AMO2Git/__init__.py | 272 +++++++++++++++++++++++++++++ AMO2Git/__main__.py | 99 +++++++++++ AMO2Git/api.py | 53 ++++++ AMO2Git/objects.py | 188 ++++++++++++++++++++ AMO2Git/utils/CommandsGenerator.py | 31 ++++ AMO2Git/utils/ExclZipFile.py | 27 +++ AMO2Git/utils/H2RequestsSession.py | 16 ++ AMO2Git/utils/__init__.py | 21 +++ AMO2Git/utils/arg.py | 21 +++ Code_Of_Conduct.md | 1 + MANIFEST.in | 4 + ReadMe.md | 47 +++++ UNLICENSE | 24 +++ pyproject.toml | 41 +++++ 20 files changed, 942 insertions(+) create mode 100644 .editorconfig create mode 100644 .github/.templateMarker create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/CI.yml create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 AMO2Git/__init__.py create mode 100644 AMO2Git/__main__.py create mode 100644 AMO2Git/api.py create mode 100644 AMO2Git/objects.py create mode 100644 AMO2Git/utils/CommandsGenerator.py create mode 100644 AMO2Git/utils/ExclZipFile.py create mode 100644 AMO2Git/utils/H2RequestsSession.py create mode 100644 AMO2Git/utils/__init__.py create mode 100644 AMO2Git/utils/arg.py create mode 100644 Code_Of_Conduct.md create mode 100644 MANIFEST.in create mode 100644 ReadMe.md create mode 100644 UNLICENSE create mode 100644 pyproject.toml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c9162b9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/.templateMarker b/.github/.templateMarker new file mode 100644 index 0000000..5e3a3e0 --- /dev/null +++ b/.github/.templateMarker @@ -0,0 +1 @@ +KOLANICH/python_project_boilerplate.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..89ff339 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: "all" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7fe33b3 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,15 @@ +name: CI +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: typical python workflow + uses: KOLANICH-GHActions/typical-python-workflow@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71fb1b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__ +*.py[co] +/*.egg-info +*.srctrlbm +*.srctrldb +build +dist +.eggs +monkeytype.sqlite3 +/.ipynb_checkpoints diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..a9a3a26 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,51 @@ +image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest + +variables: + DOCKER_DRIVER: overlay2 + SAST_ANALYZER_IMAGE_TAG: latest + SAST_DISABLE_DIND: "true" + SAST_CONFIDENCE_LEVEL: 5 + CODECLIMATE_VERSION: latest + +include: + - template: SAST.gitlab-ci.yml + - template: Code-Quality.gitlab-ci.yml + - template: License-Management.gitlab-ci.yml + +build: + tags: + - shared + - linux + stage: build + variables: + GIT_DEPTH: "1" + PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages + + before_script: + - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables` + - apt-get update + # todo: + #- apt-get -y install + #- pip3 install --upgrade + #- python3 ./fix_python_modules_paths.py + + script: + - python3 -m build -nw bdist_wheel + - mv ./dist/*.whl ./dist/AMO2Git-0.CI-py3-none-any.whl + - pip3 install --upgrade ./dist/*.whl + - coverage run --source=AMO2Git -m --branch pytest --junitxml=./rspec.xml ./tests/test.py + - coverage report -m + - coverage xml + + coverage: "/^TOTAL(?:\\s+\\d+){4}\\s+(\\d+%).+/" + + cache: + paths: + - $PYTHONUSERBASE + + artifacts: + paths: + - dist + reports: + junit: ./rspec.xml + cobertura: ./coverage.xml diff --git a/AMO2Git/__init__.py b/AMO2Git/__init__.py new file mode 100644 index 0000000..070341c --- /dev/null +++ b/AMO2Git/__init__.py @@ -0,0 +1,272 @@ +import datetime +import re +import tempfile +from collections import OrderedDict +from pathlib import Path, PurePath +from urllib.parse import urlparse + +import git + +from .objects import * +from .utils import * +from .utils.arg import argmin +from .utils.ExclZipFile import * + + +class GitName(git.Actor): + def __init__(self, name, email=None): + super().__init__(name, email) + + def __str__(self): + return name + ((" <" + email + ">") if email else "") + + +def authorGitName(author, email=None): + if not email: + email = author.email + return GitName(author.name, email) + + +class ProgressReport: + __slots__ = ("version", "processed", "total", "message") + + def __init__(self, total: int): + self.version = "" + self.processed = 0 + self.total = total + self.message = "" + + def __repr__(self): + return self.version + " " + str(self.processed / self.total * 100) + "% " + str(self.message) + + +def datetime2GitDatetimeString(dt): + """gitpython is shit, it doesn't accept datetime, dates parser is badly broken, neither dt.created.ctime(), nor dt.created.isoformat(), had to rely on ugly hacks like this function""" + return dt.replace(microsecond=0).replace(tzinfo=datetime.timezone.utc).isoformat()[:-6] + + +class SlotsDecorator: + __slots__ = tuple() + + def __init__(self, obj): + mro = [cls for cls in self.__class__.mro() if cls is not __class__] + if not hasattr(mro[1], "__slots__"): + self = obj + self.__class__ = mro[0] + #mro[0].__init__(self, *args, **kwargs) + else: + for k in dir(obj): + if not (k[0:2] == "__" and k[-2:] == "__") and not isinstance(getattr(obj.__class__, k), property) and not callable(getattr(obj.__class__, k)): + setattr(self, k, getattr(obj, k)) + + +class AddonVersionPresentInRepo(SlotsDecorator, AddonVersion): + __slots__ = ("repoTag",) + + def __init__(self, obj, tag): + SlotsDecorator.__init__(self, obj) + self.repoTag = tag + + +class AddonFileTransformer: + APP_NAME = "AMO2git" + + def __init__(self, addon, repo, downloadedAddonsDir, commiter: (GitName, str) = None, commitDate=None, tempDirPath: Path = None, authorEmail=None): + self.addon = addon + + if not authorEmail: + authorEmail = self.addon.email + self.authorEmail = authorEmail + if commiter is None: + commiter = GitName(self.__class__.APP_NAME, self.authorEmail) + elif isinstance(commiter, str): + commiter = GitName(commiter, self.authorEmail) + + self.commiter = commiter + + if commitDate is None: + commitDate = datetime2GitDatetimeString(datetime.datetime.utcnow()) # shit! + self.commitDate = commitDate + self.checkRepo(repo) + self.downloadedAddonsDir = Path(downloadedAddonsDir) + + if tempDirPath is None: + self._tempDir = tempfile.TemporaryDirectory(dir=self.downloadedAddonsDir.parent) + tempDirPath = Path(self._tempDir.name) + self.tempDirPath = tempDirPath + + versionRx = re.compile("^v(\d.+)") + + def getVersionsFromRepoTags(self): + versions = OrderedDict() + for t in self.repo.tags: + r = self.__class__.versionRx.match(t.name) + if r: + versions[r.group(1)] = t + return versions + + def matchTagsVersionsToAMO(self, versionsAMO, versionsTags): + for v, vt in versionsTags.items(): + versionsAMO[v] = AddonVersionPresentInRepo(versionsAMO[v], vt) + + def getLastVersionInRepo(self, versionsAMO): + # shit, it seems that this array is not in chronological order, need to do something! + try: + return next(reversed()) + except StopIteration: + return None + + binaryTypes = ("*.png", "*.jpg", "*.gif", "*.bmp", "*.ico", "*.so", "*.dll", "*.ocx", "*.jar", "*.zip", "*.xpi", "*.mo") + ignores = ("/META-INF",) + + def checkRepo(self, repo): + if isinstance(repo, Path): + self.repoDir = repo + try: + self.repo = git.Repo(str(self.repoDir)) + except: + self.repo = self.initRepo() + else: + self.repoDir = Path(repo.working_dir) + + def initRepo(self): + repo = git.Repo.init(str(self.repoDir)) + repo.git.lfs("track", *__class__.binaryTypes) + + with (self.repoDir / ".gitignore").open("wt", encoding="utf-8") as f: + for ign in __class__.ignores: + f.write(ign) + f.write("\n") + repo.index.add([".gitignore", ".gitattributes"]) + repo.index.commit("Initialized the repo with some files", author=self.commiter, committer=self.commiter, author_date=self.commitDate, commit_date=self.commitDate, skip_hooks=True) + repo.git.lfs("post-commit") + return repo + + def _commit(self, ver, created): + addon = ver._addon + + # self.repo.index.add(["*"]) #lfs doesn't work this way :( + self.repo.git.add("*") + + commitMsg = "" + commitMsg += ver.releaseNotes + if len(addon._authors) > 1: + commitMsg += "Authors: " + " ,".join((str(authorGitName(a)) for a in addon._authors)) + authorName = self.commiter + else: + authorName = authorGitName(addon._authors[0], self.authorEmail) + + authorDate = datetime2GitDatetimeString(created) + self.repo.index.commit(commitMsg, author=authorName, committer=self.commiter, author_date=authorDate, commit_date=self.commitDate, skip_hooks=True) + # a hack for Windows: git lfs setups hooks, but this lib cannot execute them + self.repo.git.lfs("post-commit") + + if ver and ver.version: + self.repo.create_tag("v" + ver.version) + + def downloadedFileName(self, addonFile): + return self.downloadedAddonsDir / PurePath(urlparse(addonFile.uri).path).name + + def checkFile(self, zipArch, rx, defaultFileName, text): + cands = [f for f in self.repoDir.iterdir() if rx.match(f.name)] + licFileName = None + if cands: + licFileName = cands[0] + else: + licFileName = self.repoDir / defaultFileName + + try: + zf.getinfo(licFileName.name) # check if present in the arch + except: + with licFileName.open("wt", encoding="utf-8") as f: + f.write(text) + + licenseFileNameRx = re.compile("(?:un)?license|copying(\.(?:md|markdown|a(?:scii)?doc|rst|txt))?", re.I) + + def checkLicense(self, zipArch, addonFile): + self.checkFile(zipArch, __class__.licenseFileNameRx, "License.txt", addonFile._version.licenseText) + + readmeFileNameRx = re.compile("readme(\.(?:md|markdown|a(?:scii)?doc|rst|txt))?", re.I) + + def checkReadMe(self, zipArch, addonFile): + addon = addonFile._version._addon + text = addon.summary + "\n\n" + addon.description + "\n" + self.checkFile(zipArch, __class__.readmeFileNameRx, "ReadMe.md", text) + + def unpackInternalArchive(self, archiveName, targetDir): + newArchiveName = self.tempDirPath / archiveName.name + archiveName.rename(newArchiveName) + + with ExclZipFile(newArchiveName) as zf: + zf.extractExcl(["\.git"], path=targetDir) + newArchiveName.unlink() + + def unpackInternalZipArchives(self, targetDir: Path, globExpr: str): + archives = targetDir.glob(globExpr) + names = [] + for archiveName in archives: + self.unpackInternalArchive(archiveName, targetDir) + names.append(archiveName) + return names + + def unpackInternalChromeJarsIfAny(self): + self.unpackInternalZipArchives(self.repoDir / "chrome", "*.jar") + + def unpackInternalExtensionsIfAny(self): + raise NotImplementedError() + + def getBranchName(extType: ExtensionType): + # assumming that iteration goes in the order increasing bit position and that xul is 0 + maxType = type(extType)((1 << extType.bit_length()) >> 1) + return maxType.name + + def createCommit(self, version): + for addonFile in version.files: + branchName = self.__class__.getBranchName(addonFile.extType) + if branchName not in self.repo.heads: + br = self.repo.create_head(branchName) + self.repo.head.reference = br + else: + br = self.repo.heads[branchName] + if br is not self.repo.head.reference: + prevSha = self.repo.head.reference.object.hexsha + self.repo.head.reference = br + + self.repo.head.reset(index=True, working_tree=True) + currentSha = self.repo.head.reference.object.hexsha + self.repo.git.lfs("post-checkout", prevSha, currentSha, "1") # the hook is not implemented in the lib + + with ExclZipFile(self.downloadedFileName(addonFile)) as zf: + zf.extractExcl(["\.git", "META[_-]INF"], path=self.repoDir) + self.checkLicense(zf, addonFile) + self.checkReadMe(zf, addonFile) + # TODO: split by platforms + self.unpackInternalChromeJarsIfAny() + + self._commit(version, max((addonFile.created for addonFile in version.files))) + + def transform(self): + #self.versions=self.addon._versions.sort(key=lambda v: min(v.files, key=lambda f: f.created).created) + versions = type(self.addon._versions)(reversed(self.addon._versions.items())) # AMO returns the versions in the order from the newest to the oldest, and timestamps are not always correct, old versions have the same timestamp + + self.matchTagsVersionsToAMO(versions, self.getVersionsFromRepoTags()) + + rep = ProgressReport(len(versions)) + + for v in versions.values(): + rep.version = v.version + if hasattr(v, "repoTag"): + rep.message = "skipping version: " + v.version + yield rep + else: + rep.message = "commiting version: " + v.version + yield rep + self.createCommit(v) + rep.processed += 1 + rep.message = None + yield rep + rep.message = "collecting garbage" + yield rep + self.repo.git.gc(aggressive=True) + rep.message = "Finished" + yield rep diff --git a/AMO2Git/__main__.py b/AMO2Git/__main__.py new file mode 100644 index 0000000..abb9208 --- /dev/null +++ b/AMO2Git/__main__.py @@ -0,0 +1,99 @@ +import re + +from tqdm import tqdm + +from . import * +from .utils import AMOUri2ID + +APP_NAME = "AMO2git" + +from plumbum import cli + + +class AMO2GitCLI(cli.Application): + """The main command""" + + +@AMO2GitCLI.subcommand("retrieve") +class AMO2GitRetriever(cli.Application): + """Creates a script to download all the versions of addons using aria2c""" + + streamsCount = cli.SwitchAttr("--streamsCount", int, default=32, help="Max count of streams") + versionsFolder = cli.SwitchAttr("--versionsFolder", cli.switches.MakeDirectory, default=None, help="A dir to save addons. Must be large enough.") + + def main(self, addonID: str = "noscript"): + try: + addonID = AMOUri2ID(addonID) + except: + pass + + uris = [] + versions = addonVersions(addonID) + + if not self.versionsFolder: + self.versionsFolder = addonID + "-downloads" + + self.versionsFolder = Path(self.versionsFolder) + + for ver in versions.values(): + for f in ver.files: + uris.append(f.uri) + print(genDownloadCommand(uris, self.versionsFolder, self.streamsCount)) + + +@AMO2GitCLI.subcommand("convert") +class AMO2GitConverter(cli.Application): + """Converts predownloaded versions of the addon into a git history""" + + versionsFolder = cli.SwitchAttr("--versionsFolder", cli.switches.ExistingDirectory, default=None, help="A dir to save addons. Must contain all the versions needed.") + repoFolder = cli.SwitchAttr("--repoFolder", cli.switches.MakeDirectory, default=None, help="A dir to have a git repo. Must be large enough.") + authorEmail = cli.SwitchAttr("--email", str, default=None, help="A email to use for commits") + commiterName = cli.SwitchAttr("--commiterName", str, default=AddonFileTransformer.APP_NAME, help="A name to use for a commiter of commits") + commiterEmail = cli.SwitchAttr("--commiterEmail", str, default=None, help="A email to use for a commiter of commits") + + def main(self, addonID: (str, int)): + try: + addonID = AMOUri2ID(addonID) + except: + pass + + addon = Addon(addonID) + addon.retrieveVersions() + + if not self.versionsFolder: + self.versionsFolder = addonID + "-downloads" + + if not self.repoFolder: + self.repoFolder = addonID + "-repo" + + self.versionsFolder = Path(self.versionsFolder) + self.repoFolder = Path(self.repoFolder) + + if self.commiterEmail is not None: + commiter = GitName(self.commiterName, self.commiterEmail) + else: + commiter = self.commiterName + + tr = AddonFileTransformer(addon, self.repoFolder, self.versionsFolder, authorEmail=self.authorEmail, commiter=commiter) + + progressIter = tr.transform() + progress = next(progressIter) + pr = progress.processed + + with tqdm(total=progress.total) as pb: + + def processProgress(progress, pr): + pb.total = progress.total + pb.desc = progress.version + if progress.message: + pb.write(progress.message) + pb.update(progress.processed - pr) + return progress.processed + + pr = processProgress(progress, pr) + for progress in progressIter: + pr = processProgress(progress, pr) + + +if __name__ == "__main__": + AMO2GitCLI.run() diff --git a/AMO2Git/api.py b/AMO2Git/api.py new file mode 100644 index 0000000..684687b --- /dev/null +++ b/AMO2Git/api.py @@ -0,0 +1,53 @@ +from .utils.H2RequestsSession import * + +API_BASE = "https://addons.mozilla.org/api/v4" + + +def getAddonInfoURI(addonID: (str, int)): + return API_BASE + "/addons/addon/" + str(addonID) + + +def getVersionsURI(addonID: (str, int)): + """(int:addon_id|string:addon_slug|string:addon_guid)""" + return getAddonInfoURI(addonID) + "/versions/" + + +def getVersionURI(addonID: (str, int), versionId: (str, int)): + """(int:addon_id|string:addon_slug|string:addon_guid)""" + return getVersionsURI(addonID) + "/" + str(versionId) + "/" + + +def getAuthorInfoURI(userID: (str, int)): + """(int:user_id|string:username)""" + return API_BASE + "/accounts/account/" + str(userID) + "/" + + +sess = H2RequestsSession() + + +def pagination(initialURI: (str, int)): + + t = sess.get(initialURI + "?&page_size=" + str(1 << 63)).json() + res = [] + res.extend(t["results"]) + while t["next"]: + print(t["next"]) + t = sess.get(t["next"]).json() + res.append(t) + return res + + +def getVersions(addonID: (str, int)): + return pagination(getVersionsURI(addonID)) + + +def getVersion(addonID: (str, int), versionId: (str, int)): + return sess.get(getVersionURI(addonID)).json() + + +def getAddonInfo(addonID: (str, int)): + return sess.get(getAddonInfoURI(addonID)).json() + + +def getAuthorInfo(authorId: (str, int)): + return sess.get(getAuthorInfoURI(authorId)).json() diff --git a/AMO2Git/objects.py b/AMO2Git/objects.py new file mode 100644 index 0000000..b35b64e --- /dev/null +++ b/AMO2Git/objects.py @@ -0,0 +1,188 @@ +from collections import OrderedDict +from enum import IntFlag + +import dateutil +import dateutil.parser + +from .api import * +from .utils import H2RequestsSession + + +class AddonType(IntFlag): + extension = 0 + theme = 1 + dictionary = 2 + + +class ExtensionType(IntFlag): + xul = 0 + restartless = 1 + WebExtension = 2 + + +def langSelector(field, langs=("en-US", "en")): + try: + for l in langs: + if l in field: + return field["en-US"] + else: + return next(iter(field.values())) + except: + return "" + + +class SlotsRepr: + __slots__ = tuple() + + def __init__(self): + for cls in self.__class__.mro(): + if cls == __class__: + break + for k in cls.__slots__: + setattr(self, k, None) + + def __slotsNamesIter__(self): + for cls in self.__class__.mro(): + if hasattr(cls, "__slots__"): + for k in cls.__slots__: + yield k + + def __repr__(self): + return self.__class__.__name__ + "<" + ", ".join((repr(k) + "=" + repr(getattr(self, k)) for k in self.__slotsNamesIter__() if k[0] != "_")) + ">" + + +class AMOApiItem(SlotsRepr): + __slots__ = ("_dic", "_id") + + def __init__(self, dic: dict): + super().__init__() + self._dic = dic + self._id = dic["id"] + + +class Addon(AMOApiItem): + __slots__ = ("_authors", "_versions", "name", "homepage", "email", "_type", "_supportURI") + + def __init__(self, dic: dict): + if isinstance(dic, (int, str)): + dic = getAddonInfo(dic) + + super().__init__(dic) + self._authors = [Author(a) for a in dic["authors"]] + for a in self._authors: + a.retrieveAdditional() + self.name = langSelector(self._dic["name"]) + self.homepage = langSelector(self._dic["homepage"]) + self.email = langSelector(self._dic["support_email"]) + self._supportURI = langSelector(self._dic["support_url"]) + self._dic["tags"] = set(self._dic["tags"]) + self._dic["_type"] = getattr(AddonType, self._dic["type"]) + self._versions = None + + @property + def tags(self): + return self._dic["tags"] + + def retrieveVersions(self): + self._versions = addonVersions(self) + + @property + def description(self): + return langSelector(self._dic["description"]) + + @property + def summary(self): + return langSelector(self._dic["summary"]) + + @property + def developerComments(self): + return langSelector(self._dic["developer_comments"]) + + @property + def icon(self): + return self._dic["icon_url"] + + +def addonVersions(addon: (Addon, str, int)): + if isinstance(addon, Addon): + addonId = addon._id + else: + addonId = addon + addon = None + + res = OrderedDict() + for v in getVersions(addonId): + v = AddonVersion(v, addon) + res[v.version] = v + + return res + + +class Author(AMOApiItem): + __slots__ = ("name", "username", "homepage", "location", "occupation", "_biography", "_avatar") + + def __init__(self, dic: dict, email=None): + if isinstance(dic, (int, str)): + dic = getAuthorInfo(dic) + super().__init__(dic) + self.name = dic["name"] + + def retrieveAdditional(self): + additionalInfo = getAuthorInfo(self._id) + if not additionalInfo["has_anonymous_username"]: + self.username = additionalInfo["username"] + self.homepage = additionalInfo["homepage"] + self.location = additionalInfo["location"] + self._biography = additionalInfo["biography"] + self._avatar = additionalInfo["picture_url"] + + +class AddonVersion(AMOApiItem): + __slots__ = ("_dic", "_addon", "version", "files") + + def __init__(self, dic: dict, addon=None): + if isinstance(dic, (int, str)): + if addon: + if isinstance(addon, (int, str)): + addonId = addon + else: + addonId = addon._id + dic = getVersion(addonId, dic) + else: + raise ValueError("Provide the Addon") + super().__init__(dic) + self._addon = addon + self.version = dic["version"].replace("-signed", "") + self.files = [AddonFile(f, self) for f in dic["files"]] + + @property + def releaseNotes(self): + return langSelector(self._dic["release_notes"]) + + @property + def licenseText(self): + try: + return langSelector(self._dic["license"]["text"]) + except: + return "" + + @property + def licenseName(self): + try: + return langSelector(self._dic["license"]["name"]) + except: + return "" + + +class AddonFile(AMOApiItem): + __slots__ = ("_dic", "_version", "platform", "extType", "created", "uri") + + def __init__(self, dic: dict, version=None): + if isinstance(dic, (int, str)): + raise ValueError("This dict is contained in a Version and cannot be fetched by ID") + super().__init__(dic) + self._version = version + self.extType = ExtensionType(ExtensionType.restartless * (not dic["is_restart_required"]) | ExtensionType.WebExtension * dic["is_webextension"]) + self.created = dateutil.parser.isoparse(dic["created"]) + self.platform = dic["platform"] + self.uri = dic["url"] diff --git a/AMO2Git/utils/CommandsGenerator.py b/AMO2Git/utils/CommandsGenerator.py new file mode 100644 index 0000000..6104257 --- /dev/null +++ b/AMO2Git/utils/CommandsGenerator.py @@ -0,0 +1,31 @@ +import platform +import re +import shlex + + +class CommandsGenerator: + def quote(self, input: str): + return shlex.quote(str(input)) + + def echo(self, input: str): + return "echo " + shlex.quote(str(input)) + + +specChar = re.compile("\\W") + + +class CommandsGeneratorWin(CommandsGenerator): + """A class to create shell commands. Create another class for other platforms""" + + def echo(self, input: str): + return "echo " + specChar.subn("^\\g<0>", input)[0] + + def quote(self, input: str): + # shlex.quote works incorrectly on Windows + return '"' + str(input).replace('"', '""') + '"' + + +if platform.system() == "Windows": + commandGen = CommandsGeneratorWin() +else: + commandGen = CommandsGenerator() diff --git a/AMO2Git/utils/ExclZipFile.py b/AMO2Git/utils/ExclZipFile.py new file mode 100644 index 0000000..8440d39 --- /dev/null +++ b/AMO2Git/utils/ExclZipFile.py @@ -0,0 +1,27 @@ +import re +from pathlib import Path +from zipfile import ZipFile + + +class ExclZipFile(ZipFile): + def _exclReParts(exclusions): + for excl in exclusions: + if isinstance(excl, Path): + if not excl.is_absolute(): + yield re.escape(str(excl)) + else: + raise ValueError("Paths must be relative") + elif isinstance(excl, str): + yield excl + else: + raise ValueError("Unsupported exclusion type " + type(excl).__name__) + + def _buildExclRe(exclusions): + return re.compile("^(?:(?:\.\/)*(?:\.\.\/.+)*)*(?:" + "|".join(("(?:" + p + ")" for p in __class__._exclReParts(exclusions))) + ")(?:/.+)?") + + def extractExcl(self, excls, *args, **kwargs): + _exclsRe = self.__class__._buildExclRe(excls) + + for n in self.namelist(): + if not _exclsRe.match(n): + self.extract(n, *args, **kwargs) diff --git a/AMO2Git/utils/H2RequestsSession.py b/AMO2Git/utils/H2RequestsSession.py new file mode 100644 index 0000000..790b30c --- /dev/null +++ b/AMO2Git/utils/H2RequestsSession.py @@ -0,0 +1,16 @@ +import requests + +try: + from hyper.contrib import HTTP20Adapter + + hyperHttp2Adapter = HTTP20Adapter() +except: + hyperHttp2Adapter = None + + +class H2RequestsSession(requests.Session): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if hyperHttp2Adapter: + self.mount("https://", hyperHttp2Adapter) + self.mount("http://", hyperHttp2Adapter) diff --git a/AMO2Git/utils/__init__.py b/AMO2Git/utils/__init__.py new file mode 100644 index 0000000..da6c169 --- /dev/null +++ b/AMO2Git/utils/__init__.py @@ -0,0 +1,21 @@ +import re + +from .CommandsGenerator import * + + +def genDownloadCommand(uris, destFolder, streamsCount=32, type="aria2"): + streamsCount = str(streamsCount) + + if type == "aria2": + return " ".join(("(\n" + "\n".join((commandGen.echo(uri) for uri in uris)) + "\n)", "|", "aria2c", "--continue=true", "--enable-mmap=true", "--optimize-concurrent-downloads=true", "-j", streamsCount, "-x 16", "-d", str(destFolder), "--input-file=-")) + else: + args = ["curl", "-C", "-", "--location", "--remote-name", "--remote-name-all", "--xattr"] + args.extend(uris) + return " ".join(args) + + +mozillaAddonUri = re.compile("https?://addons.mozilla.org/(?:[^\/]+/)?(?:[^\/]+/)?addon/([^\/]+)/.+") + + +def AMOUri2ID(uri: str): + return mozillaAddonUri.match(uri).group(1) diff --git a/AMO2Git/utils/arg.py b/AMO2Git/utils/arg.py new file mode 100644 index 0000000..7186bad --- /dev/null +++ b/AMO2Git/utils/arg.py @@ -0,0 +1,21 @@ +import functools + + +def arg(func): + """Transforms min, max and similar functions into the ones returning args""" + + @functools.wraps(func) + def argfunc(it, **kwargs): + keyModifier = lambda p: p[1] + modifiedKey = keyModifier + if "key" in kwargs: + if key is not None: + modifiedKey = lambda p: key(keyModifier(p)) + return func(enumerate(it), key=modifiedKey)[0] + + argfunc.__name__ = "arg" + argfunc.__name__ + return argfunc + + +argmax = arg(max) +argmin = arg(min) diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md new file mode 100644 index 0000000..bcaa2bf --- /dev/null +++ b/Code_Of_Conduct.md @@ -0,0 +1 @@ +No codes of conduct! \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..20f0fa8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include UNLICENSE +include *.md +include tests +include .editorconfig diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..cc85a47 --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,47 @@ +AMO2Git.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/) +========== +~~[wheel (GitLab)](https://gitlab.com/KOLANICH-libs/AMO2Git.py/-/jobs/artifacts/master/raw/dist/AMO2Git-0.CI-py3-none-any.whl?job=build)~~ +~~[wheel (GHA via `nightly.link`)](https://nightly.link/KOLANICH-libs/AMO2Git.py/workflows/CI/master/AMO2Git-0.CI-py3-none-any.whl)~~ +~~![GitLab Build Status](https://gitlab.com/KOLANICH-libs/AMO2Git.py/badges/master/pipeline.svg)~~ +~~![GitLab Coverage](https://gitlab.com/KOLANICH-libs/AMO2Git.py/badges/master/coverage.svg)~~ +~~[![GitHub Actions](https://github.com/KOLANICH-libs/AMO2Git.py/workflows/CI/badge.svg)](https://github.com/KOLANICH-libs/AMO2Git.py/actions/)~~ +[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH-libs/AMO2Git.py.svg)](https://libraries.io/github/KOLANICH-libs/AMO2Git.py) +[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py) + +This tool converts a release history on AMO into a git repo history. Useful when the addon developer is uncooperative. + +Prerequisites +------------- + +1. Install `aria2c`. Windows version can be downloaded [here](https://github.com/aria2/aria2/releases). +2. [Install `git`](https://git-scm.com/download) . +3. [Install `git-lfs`](https://github.com/git-lfs/git-lfs/releases). On Windows git-lfs is shipped with modern versions of [Git for Windows](https://gitforwindows.org/). +4. Install [```Python 3```](https://www.python.org/downloads/). For Windows I recommend [Anaconda](https://www.anaconda.com/download/). [```Python 2``` is dead, stop raping its corpse.](https://python3statement.org/) Use ```2to3``` with manual postprocessing to migrate incompatible code to ```3```. It shouldn't take so much time. +5. Install this tool and its dependencies: + * [```plumbum```](https://github.com/tomerfiliba/plumbum) [![PyPi Status](https://img.shields.io/pypi/v/plumbum.svg)](https://pypi.python.org/pypi/plumbum) [![TravisCI Build Status](https://travis-ci.org/tomerfiliba/plumbum.svg?branch=master)](https://travis-ci.org/tomerfiliba/plumbum) ![License](https://img.shields.io/github/license/tomerfiliba/plumbum.svg) - for command line interface + + * [```tqdm```](https://github.com/tqdm/tqdm) [![PyPi Status](https://img.shields.io/pypi/v/tqdm.svg)](https://pypi.python.org/pypi/plumbum) [![TravisCI Build Status](https://travis-ci.org/tqdm/tqdm.svg?branch=master)](https://travis-ci.org/tqdm/tqdm) ![License](https://img.shields.io/github/license/tqdm/tqdm.svg) - for progressbars + + * [```gitpython```](https://github.com/gitpython-developers/GitPython) [![PyPi Status](https://img.shields.io/pypi/v/gitpython.svg)](https://pypi.python.org/pypi/gitpython) [![TravisCI Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=master)](https://travis-ci.org/gitpython-developers/GitPython) ![License](https://img.shields.io/github/license/gitpython-developers/GitPython.svg) - for operations with git + + * [```requests```](https://github.com/requests/requests) [![PyPi Status](https://img.shields.io/pypi/v/requests.svg)](https://pypi.python.org/pypi/requests) [![TravisCI Build Status](https://travis-ci.org/requests/requests.svg?branch=master)](https://travis-ci.org/requests/requests) ![License](https://img.shields.io/github/license/requests/requests.svg) - for interacting to AMO RESTful API. + +Usage +----- + +6. Obtain an addon `id`, [`slug`](https://addons-server.readthedocs.io/en/latest/topics/api/addons.html#detail) or guid. Or you can just pass an URI, the program will parse the `slug` from it itself. +7. + * On Windows: + 1. `AMO2git retreive > ./retrieve.bat` + 2. `type ./retrieve.bat` and examine the shell script. + 3. `./retrieve.bat` + + * On Linux: + 1. `AMO2git retreive > ./retrieve.sh` + 2. `less ./retrieve.sh` and examine the shell script. + 3. `chmod +x ./retrieve.sh` + 4. `./retrieve.sh` + +8. The files will be downloaded to the subdir of the current working directory. +9. `AMO2git convert ` +The tool will create a subdir with the repo. [WebExtension](https://developer.mozilla.org/en-US/Add-ons/WebExtensions), [restartless (`bootstrap.js`)](https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Bootstrapped_extensions) and [xul](https://developer.mozilla.org/en-US/docs/Archive/Add-ons/Overlay_Extensions) versions of the addon will be in the corresponding branches. Versions will be marked with tags. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..efb9808 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..64ac44f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["setuptools>=61.2.0", "setuptools_scm[toml]>=3.4.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "AMO2Git" +readme = "ReadMe.md" +description = "This tool converts a release history on AMO into a git repo history." +authors = [{name = "KOLANICH"}] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: Public Domain", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = ["addons", "Mozilla", "xpi", "WebExtension"] +license = {text = "Unlicense"} +requires-python = ">=3.4" +dynamic = ["version"] +dependencies = [ + "requests", # @ git+https://github.com/psf/requests.git + "plumbum", # @ git+https://github.com/tomerfiliba/plumbum.git + "gitpython", # @ git+https://github.com/gitpython-developers/GitPython.git + "tqdm", # @ git+https://github.com/tqdm/tqdm.git +] + +[project.urls] +Homepage = "https://codeberg.org/KOLANICH-tools/AMO2Git.py" + +[tool.setuptools] +zip-safe = true + +[tool.setuptools.packages.find] +include = ["AMO2Git", "AMO2Git.*"] + +[tool.setuptools_scm]