Merge pull request #19 from ds-wizard/release/2.5.0

Release 2.5.0

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: ds-wizard

.gitignore

@@ -131,3 +131,5 @@ dmypy.json
 # IDE
 .idea/
 .vscode/
+
+workdir/

Dockerfile

@@ -24,4 +24,9 @@ COPY . /app
 
 RUN python setup.py install
 
-CMD ["docworker", "/app/config.yml", "/app/templates"]
+ENV DOCWORKER_CONFIG /app/config.yml
+ENV DOCWORKER_WORKDIR /tmp/docworker
+
+RUN mkdir /tmp/docworker
+
+CMD ["docworker"]

README.md

@@ -1,9 +1,11 @@
 # Data Stewardship Wizard Document Worker
 
+[![Documentation Status](https://readthedocs.org/projects/ds-wizard/badge/?version=latest)](https://docs.ds-wizard.org/en/latest/?badge=latest)
 [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/ds-wizard/document-worker)](https://github.com/ds-wizard/document-worker/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/datastewardshipwizard/document-worker)](https://hub.docker.com/r/datastewardshipwizard/document-worker)
 [![Document Worker CI](https://github.com/ds-wizard/document-worker/workflows/Document%20Worker%20CI/badge.svg?branch=master)](https://github.com/ds-wizard/document-worker/actions)
 [![GitHub](https://img.shields.io/github/license/ds-wizard/document-worker)](LICENSE)
+[![Documentation Status](https://readthedocs.org/projects/ds-wizard/badge/?version=latest)](https://docs.ds-wizard.org/en/latest/)
 
 *Worker for assembling and transforming documents*
 
@@ -14,23 +16,6 @@
 -  [wkhtmltopdf](https://github.com/wkhtmltopdf/wkhtmltopdf)
 -  [pandoc](https://github.com/jgm/pandoc)
 
-## Templates
-
-We are using HTML Jinja2 templates described by a JSON file within specified directory. The JSON file can look like this:
-
-```json
-{
-    "uuid": "4bfe909b-7dbc-40a7-8609-085e9af1df98",
-    "name": "My cool template",
-    "rootFile": "my/relative/dir/index.html.j2",
-    "wkhtmltopdf": "",
-    "pandoc": ""
-}
-```
-
-The `wkhtmltopdf` and `pandoc` fields are optional and you can specify extra command line options and arguments for calls of those commands for converting document. Path specified in `rootFile` is relative to JSON file, then paths in Jinja2 are relative to the root file.
-
-
 ## Docker
 
 Docker image is prepared with basic dependencies and worker installed. It is available though Docker Hub: [datastewardshipwizard/document-worker](https://hub.docker.com/r/datastewardshipwizard/document-worker).
@@ -46,7 +31,6 @@ $ docker build . -t docworker:local
 ### Mount points
 
 -  `/app/config.yml` = configuration file (see [example](config.yml))
--  `/app/templates` = directory with templates
 -  `/usr/share/fonts/<type>/<name>` = fonts according to [Debian wiki](https://wiki.debian.org/Fonts/PackagingPolicy) (for wkhtmltopdf)
 
 ### Fonts

config.yml

@@ -1,37 +1,39 @@
 mongo:
-  host: localhost
-  port: 27017
-  database: test_db
-  collection: documents
-  fs_collection: documentFs
+#  host: localhost
+#  port: 27017
+  database: engine-wizard
+#  collection: documents
+#  fs_collection: documentFs
+#  templates_collection: templates
+#  assets_fs_collection: templateAssetFs
 #  auth:
 #    username:
 #    password:
 #    database: (database)
 #    mechanism: SCRAM-SHA-256
 
-mq:
-  host: localhost
-  port: 5672
-  vhost: /
-  queue: test_queue
+#mq:
+#  host: localhost
+#  port: 5672
+#  vhost: /
+#  queue: document.generation
 #  auth:
 #    username:
 #    password:
 
 logging:
   level: INFO
 
-documents:
-  naming:
-    strategy: sanitize # uuid|slugify|sanitize
+#documents:
+#  naming:
+#    strategy: sanitize # uuid|slugify|sanitize
 
-externals:
-  pandoc:
-    executable: pandoc
-    args: --standalone
+#externals:
+#  pandoc:
+#    executable: pandoc
+#    args: --standalone
 #    timeout:
-  wkhtmltopdf:
-    executable: wkhtmltopdf
-    args:
+#  wkhtmltopdf:
+#    executable: wkhtmltopdf
+#    args:
 #    timeout:

docker-compose.yml

@@ -5,5 +5,4 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - './examples/templates:/app/templates:ro'
       - './config.yml:/app/config.yml:ro'

document_worker/cli.py

@@ -29,10 +29,12 @@ def validate_config(ctx, param, value: IO):
 
 
 @click.command(name='docworker')
-@click.argument('config', type=click.File('r'), callback=validate_config)
-@click.argument('templates_dir', type=click.Path(exists=True, file_okay=False, dir_okay=True))
-def main(config: DocumentWorkerConfig, templates_dir):
-    worker = DocumentWorker(config, templates_dir)
+@click.argument('config', envvar='DOCWORKER_CONFIG',
+                type=click.File('r'), callback=validate_config)
+@click.argument('workdir', envvar='DOCWORKER_WORKDIR',
+                type=click.Path(dir_okay=True, exists=True))
+def main(config: DocumentWorkerConfig, workdir: str):
+    worker = DocumentWorker(config, workdir)
     try:
         worker.run()
     except Exception as e:

document_worker/config.py

@@ -18,12 +18,15 @@ class MongoConfig:
 
     def __init__(self, host: str, port: int, username: str, password: str,
                  database: str, collection: str, fs_collection: str,
+                 templates_collection: str, assets_fs_collection: str,
                  auth_database: str, auth_mechanism: str):
         self.host = host
         self.port = port
         self.database = database
         self.collection = collection
         self.fs_collection = fs_collection
+        self.templates_collection = templates_collection
+        self.assets_fs_collection = assets_fs_collection
         self.username = username
         self.password = password
         self.auth_database = auth_database
@@ -36,6 +39,8 @@ def __str__(self):
                f'- database = {self.database} ({type(self.database)})\n' \
                f'- collection = {self.collection} ({type(self.collection)})\n' \
                f'- fs_collection = {self.fs_collection} ({type(self.fs_collection)})\n' \
+               f'- templates_collection = {self.templates_collection} ({type(self.templates_collection)})\n' \
+               f'- assets_fs_collection = {self.assets_fs_collection} ({type(self.assets_fs_collection)})\n' \
                f'- username = {self.username} ({type(self.username)})\n' \
                f'- password = {self.password} ({type(self.password)})\n' \
                f'- auth_database = {self.auth_database} ({type(self.auth_database)})\n' \
@@ -104,7 +109,7 @@ def __init__(self, level, message_format: str):
         self.message_format = message_format
 
     def __str__(self):
-        return f'MQueueConfig\n' \
+        return f'LoggingConfig\n' \
                f'- level = {self.level} ({type(self.level)})\n' \
                f'- message_format = {self.message_format} ({type(self.message_format)})\n'
 
@@ -175,6 +180,8 @@ class DocumentWorkerYMLConfigParser:
             'port': 27017,
             'collection': 'documents',
             'fs_collection': 'documentFs',
+            'templates_collection': 'templates',
+            'assets_fs_collection': 'templateAssetFs',
             MONGO_AUTH_SUBSECTION: {
                 'username': None,
                 'password': None,
@@ -274,6 +281,8 @@ def mongo(self) -> MongoConfig:
             database=self.get_or_default(self.MONGO_SECTION, 'database'),
             collection=self.get_or_default(self.MONGO_SECTION, 'collection'),
             fs_collection=self.get_or_default(self.MONGO_SECTION, 'fs_collection'),
+            templates_collection=self.get_or_default(self.MONGO_SECTION, 'templates_collection'),
+            assets_fs_collection=self.get_or_default(self.MONGO_SECTION, 'assets_fs_collection'),
             username=self.get_or_default(self.MONGO_SECTION, self.MONGO_AUTH_SUBSECTION, 'username'),
             password=self.get_or_default(self.MONGO_SECTION, self.MONGO_AUTH_SUBSECTION, 'password'),
             auth_database=self.get_or_default(self.MONGO_SECTION, self.MONGO_AUTH_SUBSECTION, 'database'),
@@ -337,6 +346,8 @@ class DocumentWorkerCFGConfigParser(configparser.ConfigParser):
             'password': None,
             'collection': 'documents',
             'fs_collection': 'documentFs',
+            'templates_collection': 'templates',
+            'assets_fs_collection': 'templateAssetFs',
             'auth_database': None,
             'auth_mechanism': 'SCRAM-SHA-256'
         },
@@ -418,6 +429,8 @@ def mongo(self) -> MongoConfig:
             database=self.get_or_default(self.MONGO_SECTION, 'database'),
             collection=self.get_or_default(self.MONGO_SECTION, 'collection'),
             fs_collection=self.get_or_default(self.MONGO_SECTION, 'fs_collection'),
+            templates_collection=self.get_or_default(self.MONGO_SECTION, 'templates_collection'),
+            assets_fs_collection=self.get_or_default(self.MONGO_SECTION, 'assets_fs_collection'),
             username=self.get_or_default(self.MONGO_SECTION, 'username'),
             password=self.get_or_default(self.MONGO_SECTION, 'password'),
             auth_database=self.get_or_default(self.MONGO_SECTION, 'auth_database'),

document_worker/consts.py

@@ -13,7 +13,7 @@ class DocumentField:
     UUID = 'uuid'
     NAME = 'name'
     STATE = 'state'
-    TEMPLATE = 'templateUuid'
+    TEMPLATE = 'templateId'
     FORMAT = 'formatUuid'
     RETRIEVED = 'retrievedAt'
     FINISHED = 'finishedAt'
@@ -22,6 +22,37 @@ class DocumentField:
     METADATA_FILENAME = 'fileName'
 
 
+class TemplateFileField:
+    FILENAME = 'fileName'
+    CONTENT = 'content'
+
+
+class TemplateAssetField:
+    UUID = 'uuid'
+    FILENAME = 'fileName'
+    CONTENT_TYPE = 'contentType'
+
+
+class FormatField:
+    UUID = 'uuid'
+    NAME = 'name'
+    STEPS = 'steps'
+
+
+class StepField:
+    NAME = 'name'
+    OPTIONS = 'options'
+
+
+class TemplateField:
+    ID = 'id'
+    NAME = 'name'
+    METAMODEL_VERSION = 'metamodelVersion'
+    FILES = 'files'
+    FORMATS = 'formats'
+    ASSETS = 'assets'
+
+
 class JobDataField:
     DOCUMENT_UUID = 'documentUuid'
     DOCUMENT_CONTEXT = 'documentContext'

document_worker/conversions.py

@@ -8,11 +8,12 @@
 from document_worker.documents import FileFormat, FileFormats
 
 
-def run_conversion(args: list, input_data: bytes, name: str,
+def run_conversion(*, args: list, workdir: str, input_data: bytes, name: str,
                    source_format: FileFormat, target_format: FileFormat, timeout=None) -> bytes:
     command = ' '.join(args)
     logging.info(f'Calling "{command}" to convert from {source_format} to {target_format}')
     p = subprocess.Popen(args,
+                         cwd=workdir,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
@@ -44,12 +45,17 @@ def __init__(self, config: DocumentWorkerConfig = None):
         self.config = config
 
     def __call__(self, source_format: FileFormat, target_format: FileFormat,
-                 data: bytes, metadata: dict) -> bytes:
+                 data: bytes, metadata: dict, workdir: str) -> bytes:
         template_args = self.extract_template_args(metadata)
         command = self.config.wkhtmltopdf.command + self.ARGS1 + template_args + self.ARGS2
         return run_conversion(
-            command, data, type(self).__name__, source_format, target_format,
-            timeout=self.config.wkhtmltopdf.timeout
+            args=command,
+            workdir=workdir,
+            input_data=data,
+            name=type(self).__name__,
+            source_format=source_format,
+            target_format=target_format,
+            timeout=self.config.wkhtmltopdf.timeout,
         )
 
     @staticmethod
@@ -63,13 +69,18 @@ def __init__(self, config: DocumentWorkerConfig = None):
         self.config = config
 
     def __call__(self, source_format: FileFormat, target_format: FileFormat,
-                 data: bytes, metadata: dict) -> bytes:
+                 data: bytes, metadata: dict, workdir: str) -> bytes:
         args = ['-f', source_format.name, '-t', target_format.name, '-o', '-']
         template_args = self.extract_template_args(metadata)
         command = self.config.pandoc.command + template_args + args
         return run_conversion(
-            command, data, type(self).__name__, source_format, target_format,
-            timeout=self.config.pandoc.timeout
+            args=command,
+            workdir=workdir,
+            input_data=data,
+            name=type(self).__name__,
+            source_format=source_format,
+            target_format=target_format,
+            timeout=self.config.pandoc.timeout,
         )
 
     @staticmethod

document_worker/templates/formats.py

@@ -2,34 +2,24 @@
 import uuid
 
 from document_worker.templates.steps import create_step, FormatStepException
+from document_worker.consts import FormatField, StepField
 from document_worker.documents import DocumentFile
 
 
-class FormatMetaField:
-    UUID = 'uuid'
-    NAME = 'name'
-    STEPS = 'steps'
-
-
-class StepMetaField:
-    NAME = 'name'
-    OPTIONS = 'options'
-
-
 class Format:
 
-    FORMAT_META_REQUIRED = [FormatMetaField.UUID,
-                            FormatMetaField.NAME,
-                            FormatMetaField.STEPS]
+    FORMAT_META_REQUIRED = [FormatField.UUID,
+                            FormatField.NAME,
+                            FormatField.STEPS]
 
-    STEP_META_REQUIRED = [StepMetaField.NAME,
-                          StepMetaField.OPTIONS]
+    STEP_META_REQUIRED = [StepField.NAME,
+                          StepField.OPTIONS]
 
     def __init__(self, template, metadata: dict):
         self.template = template
         self._verify_metadata(metadata)
-        self.uuid = uuid.UUID(metadata[FormatMetaField.UUID])
-        self.name = metadata[FormatMetaField.NAME]
+        self.uuid = uuid.UUID(metadata[FormatField.UUID])
+        self.name = metadata[FormatField.NAME]
         logging.info(f'Setting up format "{self.name}" ({self.uuid})')
         self.steps = self._create_steps(metadata)
         if len(self.steps) < 1:
@@ -39,26 +29,24 @@ def _verify_metadata(self, metadata: dict):
         for required_field in self.FORMAT_META_REQUIRED:
             if required_field not in metadata:
                 self.template.raise_exc(f'Missing required field {required_field} for format')
-        for step in metadata[FormatMetaField.STEPS]:
+        for step in metadata[FormatField.STEPS]:
             for required_field in self.STEP_META_REQUIRED:
                 if required_field not in step:
                     self.template.raise_exc(f'Missing required field {required_field} for step in format "{self.name}"')
 
     def _create_steps(self, metadata: dict):
         steps = []
-        for step_meta in metadata[FormatMetaField.STEPS]:
-            step_name = step_meta[StepMetaField.NAME]
-            step_options = step_meta[StepMetaField.OPTIONS]
+        for step_meta in metadata[FormatField.STEPS]:
+            step_name = step_meta[StepField.NAME]
+            step_options = step_meta[StepField.OPTIONS]
             try:
                 steps.append(
                     create_step(self.template.config, self.template, step_name, step_options)
                 )
             except FormatStepException as e:
-                import logging
                 logging.warning('Handling job exception', exc_info=True)
                 self.template.raise_exc(f'Cannot load step "{step_name}" of format "{self.name}": {e.message}')
             except Exception as e:
-                import logging
                 logging.warning('Handling job exception', exc_info=True)
                 self.template.raise_exc(f'Cannot load step "{step_name}" of format "{self.name}" ({e})')
         return steps