! Poetry-based deployment

! Subcategories precautions
! Poetry venv and requirements
! Poetry-based Dockerfile
- requirements.txt
+ poetry.lock

Dockerfile

@@ -1,29 +1,72 @@
-FROM python:3.11
+FROM python:3.11.6-slim as python-base
 MAINTAINER ALERT <alexey.rubasheff@gmail.com>
+# multidict doesn't support 3.12 yet
 
-# RUN apt-get update && apt-get -y install cron vim && apt-get -y clean && rm -rf /var/lib/apt/lists/*
+ENV \
+    BASE_DIR=/app \
+    SOURCE_DIR_NAME=rozetka
 
-WORKDIR /app
+WORKDIR $BASE_DIR
 
-COPY requirements.txt /app/
-RUN pip install --progress-bar=off --no-cache-dir -U pip setuptools wheel && pip install --progress-bar=off --no-cache-dir -r /app/requirements.txt
+ENV \
+    # Python
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=utf-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8 \
+    # pip
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    # poetry
+    POETRY_HOME="$BASE_DIR/poetry" \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    # venv and requirements path
+    VIRTUAL_ENV="$BASE_DIR/venv" \
+    # cache path is HOME/.cache
+    CACHE_PATH="/root/.cache" \
+    SOURCE_PATH="$BASE_DIR/$SOURCE_DIR_NAME"
 
-# COPY crontab /etc/cron.d/crontab
-# RUN chmod 0644 /etc/cron.d/crontab
+ENV PATH="$POETRY_HOME/bin:$VIRTUAL_ENV/bin:$PATH"
 
-COPY entrypoint.sh /app/
-RUN chmod +x /app/entrypoint.sh
+RUN python -m venv $VIRTUAL_ENV
 
-COPY rozetka /app/rozetka/
+ENV PYTHONPATH="$BASE_DIR:$PYTHONPATH"
 
-# RUN /usr/bin/crontab /etc/cron.d/crontab
 
-ENV PYTHONIOENCODING=utf-8
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US.UTF-8
-ENV PYTHONUNBUFFERED=1
+FROM python-base as builder-base
 
-# run crond as main process of container
-# CMD ["cron", "-f"]
-CMD ["/app/entrypoint.sh"]
+RUN apt-get update && \
+    apt-get install -y curl
+
+RUN --mount=type=cache,target=$CACHE_PATH \
+    curl -sSL https://install.python-poetry.org | python -
+
+WORKDIR $BASE_DIR
+
+COPY poetry.lock pyproject.toml ./
+
+RUN --mount=type=cache,target=$CACHE_PATH \
+    poetry install --no-root --only main
+
+
+FROM builder-base as development
+
+WORKDIR $BASE_DIR
+
+RUN --mount=type=cache,target=$CACHE_PATH \
+    poetry install --no-root
+
+CMD ["bash"]
+
+
+FROM python-base as production
+
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $VIRTUAL_ENV $VIRTUAL_ENV
+
+WORKDIR $BASE_DIR
+
+COPY poetry.lock pyproject.toml ./
+COPY $SOURCE_DIR_NAME ./$SOURCE_DIR_NAME/
+
+CMD ["sh", "-c", "python -m $SOURCE_DIR_NAME.runners.parse_api"]

HISTORY.rst

@@ -3,6 +3,14 @@ History
 =======
 
 
+1.1.8 (2023-10-20)
+------------------
+
+* Subcategories precautions
+* Poetry venv and requirements
+* Poetry-based Dockerfile
+
+
 1.1.7 (2023-08-27)
 ------------------
 

poetry_publish.cmd

@@ -1,6 +1,6 @@
 
 @echo off
 pushd %~dp0
-call .venv\scripts\activate
+rem call .venv\scripts\activate
 poetry config repositories.test-pypi https://test.pypi.org/legacy/
 poetry publish -r test-pypi --build

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rozetka-api"
-version = "1.1.7"
+version = "1.1.8"
 description = "Rozetka Python API"
 authors = ["Alexey ALERT Rubasheff <alexey.rubasheff@gmail.com>"]
 readme = "README.rst"
@@ -21,21 +21,21 @@ include = ["HISTORY.rst"]
 generate-setup-file = true
 
 [tool.poetry.dependencies]
-python = "^3.10"
-pip = "^22.2.2"
-requests = "^2.28.1"
-pathlib = "^1.0.1"
-pendulum = "^2.1.2"
-influxdb = {extras = ["async"], version = "^5.3.1"}
-aiohttp-retry = "^2.8.3"
-global-logger = "^0.3.30"
-progress = "^1.6"
-ratelimit = "^2.2.1"
-python-worker = "^1.10.1"
-knockknock = "^0.1.8.1"
+python = "^3.11.6,<3.12"
+pip = "*"
+requests = "*"
+pathlib = "*"
+pendulum = "*"
+aiohttp-retry = "*"
+global-logger = "*"
+progress = "*"
+ratelimit = "*"
+python-worker = "*"
+knockknock = "*"
+influxdb-client = {version = "*", extras = ["async"]}
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.1.3"
+pytest = "*"
 
 [tool.poetry-dynamic-versioning]
 enable = true

rozetka/entities/category.py

@@ -262,6 +262,7 @@ def _get_subcategories(self, parent_key='parent_id'):
                     true_cat = Category.get(parent_category_id)
                     true_cat.subcategories_data = true_cat.subcategories_data or []
                     true_cat.subcategories_data.append(pop)
+                    true_cat.subcategories_data = [_ for _ in true_cat.subcategories_data if isinstance(_, dict)]
                     true_cat.subcategories_data.sort(key=lambda i: i.get('id', 0))
                     true_subcategories_data = []
                     for data in true_cat.subcategories_data:

rozetka/entities/supercategory.py

@@ -211,7 +211,9 @@ def subcategories_data(self):
 
             category_tree: dict = category_trees[0]
             content: dict = category_tree.get('content', dict())
-            self._subcategories_data = output = content.get('items', list())
+            output = content.get('items', list())
+            output = [_ for _ in output if isinstance(_, dict)]
+            self._subcategories_data = output
             LOG.debug(f"Got {len(output)} subcategories data for {self}")
         return self._subcategories_data
 
@@ -226,8 +228,9 @@ def subcategories(self):
 
 if __name__ == '__main__':
     LOG.verbose = False
-    all_items_ = get_all_items_recursively()
-    supercategory = SuperCategory.get(2394287)
+    # all_items_ = get_all_items_recursively()
+    supercategory = SuperCategory.get(4625734)
+    supercategory.subcategories_data
     iids = supercategory.items_ids
     supers = get_super_categories()
     ac = list(get_all_categories_recursively())