Merge branch 'master' into bko-bencher

.github/workflows/check-runtime-migration.yml

@@ -34,7 +34,7 @@ jobs:
   # rococo and westend are disabled for now (no access to parity-chains.parity.io)
   check-runtime-migration:
     runs-on: arc-runners-polkadot-sdk-beefy
-    timeout-minutes: 30
+    timeout-minutes: 40
     needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}

.github/workflows/checks-quick.yml

@@ -117,30 +117,30 @@ jobs:
           CONFIG: .github/.markdownlint.yaml
         run: |
           echo "Checking markdown formatting. More info: docs/contributor/markdown_linting.md"
+          echo "To fix potential erros, you can run 'markdownlint --config .github/.markdownlint.yaml -f --ignore target .' locally."
           markdownlint --config "$CONFIG" --ignore target .
   check-umbrella:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
       - name: install python deps
-        run: |
-          sudo apt-get update && sudo apt-get install -y python3-pip python3
-          pip3 install "cargo-workspace>=1.2.4" toml
+        run: pip3 install "cargo-workspace>=1.2.4" toml
       - name: check umbrella correctness
         run: |
+          # Fixes "detected dubious ownership" error in the ci
+          git config --global --add safe.directory '*'
           python3 scripts/generate-umbrella.py --sdk . --version 0.1.0
           cargo +nightly fmt --all
+
           if [ -n "$(git status --porcelain)" ]; then
             cat <<EOF
-          👋 Hello developer! The SemVer information that you declared in the prdoc file did not match what the CI detected.
+          👋 Hello developer! Apparently you added a new crate that is not part of the umbrella crate?
 
-          Please check the output above and see the following links for more help:
-          - https://github.com/paritytech/polkadot-sdk/blob/master/docs/contributor/prdoc.md#record-semver-changes
-          - https://forum.polkadot.network/t/psa-polkadot-sdk-to-use-semver
+          You can just apply the patch (git apply PATCH_NAME) that was printed to make this CI check succeed.
 
           Otherwise feel free to ask in the Merge Request or in Matrix chat.
           EOF

.github/workflows/release-50_publish-docker.yml

@@ -27,6 +27,7 @@ on:
         options:
           - polkadot
           - polkadot-parachain
+          - chain-spec-builder
 
       release_id:
         description: |
@@ -74,7 +75,7 @@ env:
 
 jobs:
   fetch-artifacts: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build
-    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }}
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'chain-spec-builder' || inputs.image_type == 'rc' }}
     runs-on: ubuntu-latest
 
     steps:
@@ -97,7 +98,7 @@ jobs:
 
       - name: Fetch rc artifacts or release artifacts from s3 based on version
         #this step runs only if the workflow is triggered manually
-        if: ${{ env.EVENT_NAME  == 'workflow_dispatch' }}
+        if: ${{ env.EVENT_NAME  == 'workflow_dispatch' && inputs.binary != 'chain-spec-builder'}}
         run: |
           . ./.github/scripts/common/lib.sh
 
@@ -106,15 +107,22 @@ jobs:
 
           fetch_release_artifacts_from_s3
 
-      - name: Cache the artifacts
-        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
+      - name: Fetch chain-spec-builder rc artifacts or release artifacts based on release id
+        #this step runs only if the workflow is triggered manually and only for chain-spec-builder
+        if: ${{ env.EVENT_NAME  == 'workflow_dispatch' && inputs.binary == 'chain-spec-builder' }}
+        run: |
+          . ./.github/scripts/common/lib.sh
+          RELEASE_ID=$(check_release_id "${{ inputs.release_id }}")
+          fetch_release_artifacts
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
-          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
-          path: |
-            ./release-artifacts/${{ env.BINARY }}/**/*
+          name: release-artifacts
+          path: release-artifacts/${{ env.BINARY }}/**/*
 
   build-container: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build
-    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }}
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'chain-spec-builder' || inputs.image_type == 'rc' }}
     runs-on: ubuntu-latest
     needs: fetch-artifacts
     environment: release
@@ -123,26 +131,23 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Get artifacts from cache
-        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
-        with:
-          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
-          fail-on-cache-miss: true
-          path: |
-            ./release-artifacts/${{ env.BINARY }}/**/*
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
 
       - name: Check sha256 ${{ env.BINARY }}
-        working-directory: ./release-artifacts/${{ env.BINARY }}
+        if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'polkadot' }}
+        working-directory: release-artifacts
         run: |
-          . ../../.github/scripts/common/lib.sh
+          . ../.github/scripts/common/lib.sh
 
           echo "Checking binary $BINARY"
           check_sha256 $BINARY && echo "OK" || echo "ERR"
 
       - name: Check GPG ${{ env.BINARY }}
-        working-directory: ./release-artifacts/${{ env.BINARY }}
+        if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'polkadot' }}
+        working-directory: release-artifacts
         run: |
-          . ../../.github/scripts/common/lib.sh
+          . ../.github/scripts/common/lib.sh
           import_gpg_keys
           check_gpg $BINARY
 
@@ -164,20 +169,21 @@ jobs:
           echo "No tag, doing without"
 
       - name: Fetch release tags
-        working-directory: ./release-artifacts/${{ env.BINARY }}
+        working-directory: release-artifacts
         if: ${{ env.IMAGE_TYPE == 'release'}}
         id: fetch_release_refs
         run: |
           chmod a+rx $BINARY
-          VERSION=$(./$BINARY --version | awk '{ print $2 }' )
+          [[ $BINARY != 'chain-spec-builder' ]] && VERSION=$(./$BINARY --version | awk '{ print $2 }' )
+
           release=$( echo $VERSION | cut -f1 -d- )
           echo "tag=latest" >> $GITHUB_OUTPUT
           echo "release=${release}" >> $GITHUB_OUTPUT
 
-      - name: Build Injected Container image for polkadot rc
-        if: ${{ env.BINARY == 'polkadot' }}
+      - name: Build Injected Container image for polkadot rc or chain-spec-builder
+        if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'chain-spec-builder' }}
         env:
-          ARTIFACTS_FOLDER: ./release-artifacts
+          ARTIFACTS_FOLDER: release-artifacts
           IMAGE_NAME: ${{ env.BINARY }}
           OWNER: ${{ env.DOCKER_OWNER }}
           TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
@@ -189,7 +195,7 @@ jobs:
       - name: Build Injected Container image for polkadot-parachain
         if: ${{ env.BINARY == 'polkadot-parachain' }}
         env:
-          ARTIFACTS_FOLDER: ./release-artifacts
+          ARTIFACTS_FOLDER: release-artifacts
           IMAGE_NAME: ${{ env.BINARY }}
           OWNER: ${{ env.DOCKER_OWNER }}
           DOCKERFILE: docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile
@@ -219,7 +225,11 @@ jobs:
           RELEASE_TAG: ${{ steps.fetch_rc_refs.outputs.release || steps.fetch_release_refs.outputs.release  }}
         run: |
           echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}"
-          $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version
+          if [[ ${BINARY} == 'chain-spec-builder' ]]; then
+            $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG}
+          else
+            $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version
+          fi
 
   fetch-latest-debian-package-version: # this job will be triggered for polkadot release build
     if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }}

.gitlab/pipeline/check.yml

@@ -24,30 +24,6 @@ check-try-runtime:
     # experimental code may rely on try-runtime and vice-versa
     - time cargo check --locked --all --features try-runtime,experimental
 
-# FIXME
-.cargo-deny-licenses:
-  stage: check
-  extends:
-    - .docker-env
-    - .test-pr-refs
-  variables:
-    CARGO_DENY_CMD: "cargo deny --all-features check licenses -c ./substrate/scripts/ci/deny.toml"
-  script:
-    - $CARGO_DENY_CMD --hide-inclusion-graph
-  after_script:
-    - echo "___The complete log is in the artifacts___"
-    - $CARGO_DENY_CMD 2> deny.log
-    - if [ $CI_JOB_STATUS != 'success' ]; then
-      echo 'Please check license of your crate or add an exception to scripts/ci/deny.toml';
-      fi
-  allow_failure: true
-  artifacts:
-    name: $CI_COMMIT_SHORT_SHA
-    expire_in: 3 days
-    when: always
-    paths:
-      - deny.log
-
 # from substrate
 # not sure if it's needed in monorepo
 check-dependency-rules: