diff --git a/.github/workflows/allure_report.yaml b/.github/workflows/allure_report.yaml new file mode 100644 index 000000000..10a0cb799 --- /dev/null +++ b/.github/workflows/allure_report.yaml @@ -0,0 +1,99 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: Allure Report Generation + +on: [workflow_call] + +jobs: + allure-report: + name: Publish Allure report + runs-on: ubuntu-latest + timeout-minutes: 5 + if: always() && !cancelled() + steps: + - name: Download Allure + # Following instructions from https://allurereport.org/docs/install-for-linux/#install-from-a-deb-package + run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install Allure + run: | + sudo apt-get update + sudo apt-get install ./allure_*.deb -y + - name: Checkout GitHub pages branch + uses: actions/checkout@v4 + with: + ref: gh-pages + path: repo/ + # The `gh-pages` branch is used to host the Allure report site. + # With every workflow run, the workflow creates a new folder with the run_number and stores the test results there + - name: Download fallback test results + uses: actions/download-artifact@v4 + with: + path: allure-collection-fallback-results/ + pattern: allure-fallback-results* + merge-multiple: true + - name: Download actual test results + uses: actions/download-artifact@v4 + with: + path: allure-results/ + pattern: allure-results* + merge-multiple: true + - name: Install CLI + run: pipx install git+https://github.com/canonical/data-platform-workflows@v24.0.0#subdirectory=python/cli + - name: Combine Allure fallback results & actual results + # For every test: if actual result available, use that. Otherwise, use fallback result + # So that, if actual result not available, Allure report will show "unknown"/"failed" test result + # instead of omitting the test + run: allure-add-default-for-missing-results --allure-results-dir=allure-results --allure-collection-default-results-dir=allure-collection-fallback-results + - name: Load test report history + run: | + if [[ -d repo/_latest/history/ ]] + then + echo 'Loading history' + cp -r repo/_latest/history/ allure-results/ + fi + - name: Create executor.json + shell: python + run: | + # Reverse engineered from https://github.com/simple-elf/allure-report-action/blob/eca283b643d577c69b8e4f048dd6cd8eb8457cfd/entrypoint.sh + # Not using the original action due to security concerns over using 3rd party github actions and the risk of running arbitrary code + import json + + DATA = { + "name": "GitHub Actions", + "type": "github", + "buildOrder": ${{ github.run_number }}, # TODO future improvement: use run ID + "buildName": "Run ${{ github.run_id }}", + "buildUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "reportUrl": "../${{ github.run_number }}/", + } + with open("allure-results/executor.json", "w") as file: + json.dump(DATA, file) + - name: Generate Allure report + run: allure generate + - name: Create index.html + shell: python + run: | + DATA = f""" + + + + """ + with open("repo/index.html", "w") as file: + file.write(DATA) + - name: Update GitHub pages branch + working-directory: repo/ + run: | + mkdir '${{ github.run_number }}' + rm -f _latest + ln -s '${{ github.run_number }}' _latest + cp -r ../allure-report/. _latest/ + git add . + git config user.name "GitHub Actions" + # user.email obtained from https://github.com/actions/checkout/issues/13#issuecomment-724415212 + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Allure report ${{ github.run_number }}" + # Uses token set in checkout step + git push origin gh-pages diff --git a/.github/workflows/integration_test_run.yaml b/.github/workflows/integration_test_run.yaml index f66a7c1ad..0f8566307 100644 --- a/.github/workflows/integration_test_run.yaml +++ b/.github/workflows/integration_test_run.yaml @@ -171,6 +171,69 @@ jobs: )) || inputs.runs-on }} steps: + - uses: actions/checkout@v4.2.2 + - name: Integration tests variable setting + run: | + allure_artifact_suffix=$(uuidgen) + series="" + if [ ! -z ${{ matrix.series }} ]; then + series="--series ${{ matrix.series }}" + allure_artifact_suffix="$allure_artifact_suffix"-${{ matrix.series }} + fi + echo "SERIES=$series" >> $GITHUB_ENV + module="" + if [ ! -z ${{ matrix.modules }} ]; then + module="-k ${{ matrix.modules }}" + allure_artifact_suffix="$allure_artifact_suffix"-${{ matrix.modules }} + fi + echo "MODULE=$module" >> $GITHUB_ENV + echo "ALLURE_ARTIFACT_SUFFIX=$allure_artifact_suffix" >> $GITHUB_ENV + - name: Install tox + run: | + if which tox &> /dev/null; then + echo "tox is already installed." + tox --version + fi + + pip_path=$(which pip 2>/dev/null) + SYSTEM_PIP_PATH="/usr/bin/pip" + if [ -n "$pip_path" ] && [ "$pip_path" != "$SYSTEM_PIP_PATH" ]; then + echo "Pip is available and not system-managed. Installing tox" + pip install tox + fi + + if which pipx &> /dev/null; then + echo "Pipx is available. Installing tox" + pipx install tox + fi + + echo "Neither pip nor pipx are available. Installing pipx via apt..." + sudo apt-get update -yqq + sudo apt-get install -yqq pipx + pipx ensurepath + sudo pipx ensurepath + + echo "Installing tox with pipx..." + pipx install tox + - name: Check Allure + working-directory: ${{ inputs.working-directory }} + run: | + tox -e ${{ inputs.test-tox-env }} --notest --list-dependencies 2>&1 + tox -e ${{ inputs.test-tox-env }} --notest --list-dependencies 2>&1 | grep -q allure \ + && echo 'ENABLE_ALLURE=true' >> $GITHUB_ENV \ + || echo 'ENABLE_ALLURE=false' >> $GITHUB_ENV + - name: Collect tests for Allure + if: env.ENABLE_ALLURE == 'true' + working-directory: ${{ inputs.working-directory }} + run: | + tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} --allure-collection-dir=allure-default ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + - name: Upload Default Allure results + if: env.ENABLE_ALLURE == 'true' + timeout-minutes: 3 + uses: actions/upload-artifact@v4 + with: + name: allure-fallback-results-${{ env.ALLURE_ARTIFACT_SUFFIX }} + path: ${{ inputs.working-directory }}allure-default/ - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: @@ -266,14 +329,12 @@ jobs: VAULT_APPROLE_SECRET_ID: ${{ secrets.VAULT_APPROLE_SECRET_ID }} - run: sudo apt install skopeo -y - - name: Plan Integration uses: canonical/operator-workflows/internal/plan-integration@main id: plan-integration with: plan: ${{ inputs.plan }} - - - name: Integration tests variable setting + - name: Charm name setting working-directory: ${{ inputs.working-directory }}/${{ inputs.charm-directory }} run: | CHARM_NAME="$([ -f metadata.yaml ] && yq '.name' metadata.yaml || echo UNKNOWN)" @@ -284,26 +345,31 @@ jobs: args="${{ steps.plan-integration.outputs.args }}" echo "ARGS=$args" >> $GITHUB_ENV - series="" - if [ ! -z ${{ matrix.series }} ]; then - series="--series ${{ matrix.series }}" - fi - echo "SERIES=$series" >> $GITHUB_ENV - module="" - if [ ! -z ${{ matrix.modules }} ]; then - module="-k ${{ matrix.modules }}" - fi - echo "MODULE=$module" >> $GITHUB_ENV - name: Run k8s integration tests working-directory: ${{ inputs.working-directory }} if: ${{ inputs.provider == 'microk8s' }} run: | - tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + if [ "$ENABLE_ALLURE" == "true" ]; then + tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --alluredir=allure-results ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + else + tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + fi - name: Run lxd integration tests working-directory: ${{ inputs.working-directory }} if: ${{ inputs.provider == 'lxd' }} run: | - tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + if [ "$ENABLE_ALLURE" == "true" ]; then + tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --alluredir=allure-results ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + else + tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} + fi + - name: Upload Allure results + timeout-minutes: 3 + if: env.ENABLE_ALLURE == 'true' && always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-${{ env.ALLURE_ARTIFACT_SUFFIX }} + path: ${{ inputs.working-directory }}allure-results/ - name: Tmate debugging session (self-hosted) if: ${{ failure() && (inputs.tmate-debug || runner.debug) && inputs.self-hosted-runner }} uses: canonical/action-tmate@main diff --git a/.github/workflows/workflow_test.yaml b/.github/workflows/workflow_test.yaml index 631b45114..88425cab4 100644 --- a/.github/workflows/workflow_test.yaml +++ b/.github/workflows/workflow_test.yaml @@ -103,6 +103,14 @@ jobs: integration-test-workflow-file: workflow_test.yaml working-directory: tests/workflows/integration/test-upload-charm/ workflow-run-id: ${{ github.run_id }} + allure-report: + if: always() && !cancelled() + needs: + - integration + - integration-juju3 + - integration-artifact + - integration-self-hosted + uses: ./.github/workflows/allure_report.yaml check: runs-on: ubuntu-latest if: always() && !cancelled() @@ -119,6 +127,7 @@ jobs: - integration-craft - publish - publish-artifact + - allure-report steps: - run: | [ '${{ needs.simple.result }}' = 'success' ] || (echo simple failed && false) @@ -132,3 +141,5 @@ jobs: [ '${{ needs.integration-craft.result }}' = 'success' ] || (echo integration-craft failed && false) [ '${{ needs.publish.result }}' != 'failure' ] || (echo publish failed && false) [ '${{ needs.publish-artifact.result }}' != 'failure' ] || (echo publish failed && false) + [ '${{ needs.allure-report.result }}' != 'failure' ] || (echo allure-report failed && false) + diff --git a/docs/how-to/integration_test_allure.md b/docs/how-to/integration_test_allure.md new file mode 100644 index 000000000..1ce16787b --- /dev/null +++ b/docs/how-to/integration_test_allure.md @@ -0,0 +1,64 @@ +# How to set up Allure Reports for integration tests + +This how-to guide describes how to integrate [Allure Reports](https://allurereport.org/) into your code repository's [integration_test_run.yaml](https://github.com/canonical/operator-workflows?tab=readme-ov-file#integration-test-workflow-canonicaloperator-workflowsgithubworkflowsintegration_testyamlmain). + +## Adding allure-pytest and pytest collection plugin + +Include the following snippet in the `requirements.txt` that is called by the integration test: + +``` +allure-pytest>=2.8.18 +``` + +Add the following line under the dependencies (`deps`) in the integration section inside `tox.ini`: + +``` +git+https://github.com/canonical/operator-workflows@main\#subdirectory=python/pytest_plugins/allure_pytest_collection_report +``` + +## Calling the allure-workflow + +To call the reusable workflow [allure_report.yaml](https://github.com/canonical/operator-workflows/blob/main/.github/workflows/allure_report.yaml), add the following lines at the end of the workflow that runs the integrations tests: + +``` + allure-report: + if: always() && !cancelled() + needs: + - [list of jobs that call integration_test workflow whose tests you would like to visualize] + uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main +``` + +For an example of this implementation, see [the GitHub runner repository](https://github.com/canonical/github-runner-operator/pull/412). + +**NOTE:** If the workflow is being called inside a matrix with the same test modules run with different parameters, the Allure Report will only display the results of the last combination. + +## Changing branch permissions + +**NOTE:** For this step, you need admin access to the repository. + +If your repository is configured to have signed commits for all branches by default, you need to create a seperate protection rule for the `gh-pages` branch with the signed commits disabled. + +- Go to the repository's **Settings > Branches** and next to Branch protection rules, select **Add rule** +- Enter the branch name **gh-pages** and click **Save changes** (Ensure that "require signed commits" is unchecked) + +## Github pages branch + +- Create `gh-pages` branch: + +``` +# For first run, manually create branch with no history + # (e.g. + # git checkout --orphan gh-pages + # git rm -rf . + # touch .nojekyll + # git add .nojekyll + # git commit -m "Initial commit" + # git push origin gh-pages + # ) + ``` + + - Enable GitHub pages publishing at ** Settings > Pages ** and set branch name as `gh-pages`: + +image + +For an example of the first two steps, see [the GitHub runner repository](https://github.com/canonical/github-runner-operator/pull/412). diff --git a/tests/workflows/integration/test-rock/requirements.txt b/tests/workflows/integration/test-rock/requirements.txt index 2d81d3bb6..0d05e6c9d 100644 --- a/tests/workflows/integration/test-rock/requirements.txt +++ b/tests/workflows/integration/test-rock/requirements.txt @@ -1 +1,2 @@ ops +allure-pytest>=2.8.18 diff --git a/tests/workflows/integration/test-upload-charm/requirements.txt b/tests/workflows/integration/test-upload-charm/requirements.txt index 56f5f6428..f9239509b 100644 --- a/tests/workflows/integration/test-upload-charm/requirements.txt +++ b/tests/workflows/integration/test-upload-charm/requirements.txt @@ -1 +1,2 @@ ops >= 1.5.0 +allure-pytest>=2.8.18 \ No newline at end of file diff --git a/tests/workflows/integration/test-upload-charm/tox.ini b/tests/workflows/integration/test-upload-charm/tox.ini index 1a868e5b6..04a96ee43 100644 --- a/tests/workflows/integration/test-upload-charm/tox.ini +++ b/tests/workflows/integration/test-upload-charm/tox.ini @@ -112,6 +112,7 @@ deps = macaroonbakery==1.3.2 websockets<14.0 # https://github.com/juju/python-libjuju/issues/1184 -r{toxinidir}/requirements.txt + git+https://github.com/canonical/data-platform-workflows@v24.0.0\#subdirectory=python/pytest_plugins/allure_pytest_collection_report commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} @@ -126,5 +127,6 @@ deps = macaroonbakery==1.3.2 websockets<14.0 # https://github.com/juju/python-libjuju/issues/1184 -r{toxinidir}/requirements.txt + git+https://github.com/canonical/data-platform-workflows@v24.0.0\#subdirectory=python/pytest_plugins/allure_pytest_collection_report commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs}