Merge pull request #398 from 4dn-dcic/fix-publish

Fix for GA publish and CI builds for Python 3.8, 3.9, 3.10, and 3.11

.github/workflows/main-publish.yml

@@ -32,5 +32,4 @@ jobs:
           PYPI_USER: ${{ secrets.PYPI_USER }}
           PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: | 
-          make configure
           make publish-for-ga

.github/workflows/main.yml

@@ -14,22 +14,117 @@ on:
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+# In this case, since this repo (tibanna) has tests which cannot reliablly run concurrently,
+# because (at least) they write/read to/from fixed locations in S3, we need to run the tests
+# for each of the Python versions serially; so the build_<version> sections below are just
+# clones of each other, with the proper version and "needs" clause on the previous build.
+# Using the strategy/matrix mechanism as is done in other repos causes the tests to run
+# concurrently. TODO: Find a way to do this without having to clone these sections.
 jobs:
-  # This workflow contains a single job called "build"
-  build:
+  build_3_8:
     # The type of runner that the job will run on
     runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        python_version: ['3.11']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python_version }}
+          python-version: '3.8'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Deps
+        run: |
+          make install
+          
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: QA (unit)
+        run: |
+          poetry run invoke test --no-flake
+
+  build_3_9:
+    needs: build_3_8
+    # The type of runner that the job will run on
+    runs-on: ubuntu-22.04
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Deps
+        run: |
+          make install
+          
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: QA (unit)
+        run: |
+          poetry run invoke test --no-flake
+
+  build_3_10:
+    needs: build_3_9
+    # The type of runner that the job will run on
+    runs-on: ubuntu-22.04
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Deps
+        run: |
+          make install
+          
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: QA (unit)
+        run: |
+          poetry run invoke test --no-flake
+
+  build_3_11:
+    needs: build_3_10
+    # The type of runner that the job will run on
+    runs-on: ubuntu-22.04
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
 
       - name: Install Poetry
         uses: snok/install-poetry@v1

CHANGELOG.rst

@@ -4,6 +4,19 @@ Change Log
 ==========
 
 
+5.1.0
+=====
+* Fixed GA publish which was not working because it could not find dcicutils because
+  it is not a dependency in pyproject.toml; workaround is to call it manually via straight
+  python and not via pyproject.toml script. N.B. HOWEVER: It still does not work because
+  pypi project credentials are not yet defined for this repo.
+* Added Python 3.8, 3.9, 3.10, as well as 3.11 for GA CI build. This causes failures
+  because of the way the tests were written - writing/reading to/from fixed location in S3,
+  which means that concurrent runs do not reliably succeed, i.e. because they are stomping
+  on each other. Workaround was to define separate build steps (cloned for now) in main.yml
+  with appropriate "needs" clauses which forces them to execute serially.
+
+
 5.0.0
 =====
 

Makefile

@@ -29,11 +29,14 @@ publish:
 	poetry run publish-to-pypi
 
 publish-for-ga:
-	# Need this poetry install first for some reason in GitHub Actions, otherwise getting this:
-	# Warning: 'publish-to-pypi' is an entry point defined in pyproject.toml, but it's not installed as a script. You may get improper `sys.argv[0]`.
-	# Only a warning, but then it does not find dcicutils for some reason.
-	poetry install
-	poetry run publish-to-pypi --noconfirm
+	# Normally this is done like this:
+	# -> poetry run publish-to-pypi --noconfirm
+	# But this will not work in GA because this repo (tibanna) does not have dcicutils as a
+	# dependency in pyproject.toml and the publish-to-pypi is defined within pyproject.toml
+	# so it will not find dcicutils, even we install it explicitly here (or in the GA yml file),
+	# so we just call the script manually here, just like it is defined in pyproject.toml.
+	pip install dcicutils poetry
+	python -m dcicutils.scripts.publish_to_pypi --noconfirm
 
 publish-pypi:
 	scripts/publish-pypi

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tibanna"
-version = "5.0.0"
+version = "5.1.0"
 description = "Tibanna runs portable pipelines (in CWL/WDL) on the AWS Cloud."
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"