feat: execute behave, pytest and nose2 tests in PRs

.github/workflows/ci.yml

@@ -1,27 +1,213 @@
 name: build
 
-on: [push, pull_request]
+on:
+  pull_request:
 
 jobs:
-  test:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up python 3.11
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.11'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements_dev.txt
+    - name: Lint and format check with ruff
+      run: |
+        ruff check .
+        ruff format --check .
+
+  behave-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.13']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
       fail-fast: false
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements_dev.txt
+    - name: Run tests
+      env:
+        TOOLIUM_DRIVER_HEADLESS: Driver_headless=true
+      run: |
+        behave web_behave/features/ --junit --junit-directory web_behave/output/reports
+      continue-on-error: true
+    - name: Upload output folder
+      uses: actions/upload-artifact@v7
+      if: always()
+      with:
+        name: behave-tests-output-${{ matrix.python-version }}
+        path: web_behave/output
+    - name: Publish test results
+      uses: dorny/test-reporter@v3
+      if: always()
+      with:
+        name: behave tests results (${{ matrix.python-version }})
+        path: web_behave/output/reports/*.xml
+        reporter: java-junit
+        fail-on-error: true
 
+  pytest-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+      fail-fast: false
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+    - uses: actions/checkout@v6
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r requirements_dev.txt
-    - name: Lint and format check with ruff
+    - name: Run tests
+      env:
+        TOOLIUM_DRIVER_HEADLESS: Driver_headless=true
       run: |
-        ruff check .
-        ruff format --check .
+        cd web_pytest
+        python -m pytest --junitxml=output/reports/junit-pytest.xml
+      continue-on-error: true
+    - name: Upload output folder
+      uses: actions/upload-artifact@v7
+      if: always()
+      with:
+        name: pytest-tests-output-${{ matrix.python-version }}
+        path: web_pytest/output
+    - name: Publish test results
+      uses: dorny/test-reporter@v3
+      if: always()
+      with:
+        name: pytest tests results (${{ matrix.python-version }})
+        path: web_pytest/output/reports/*.xml
+        reporter: java-junit
+        fail-on-error: true
+
+  nose2-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements_dev.txt
+    - name: Run tests
+      env:
+        TOOLIUM_DRIVER_HEADLESS: Driver_headless=true
+      run: |
+        mkdir -p web_nose2/output/reports
+        python -m nose2 web_nose2 -A '!local' --junit-xml-path web_nose2/output/reports/junit-nose.xml || true
+      continue-on-error: true
+    - name: Upload output folder
+      uses: actions/upload-artifact@v7
+      if: always()
+      with:
+        name: nose2-tests-output-${{ matrix.python-version }}
+        path: web_nose2/output
+    - name: Publish test results
+      uses: dorny/test-reporter@v3
+      if: always()
+      with:
+        name: nose2 tests results (${{ matrix.python-version }})
+        path: web_nose2/output/reports/*.xml
+        reporter: java-junit
+        fail-on-error: true
+
+  playwright-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements_dev.txt
+        pip install toolium[playwright]
+        playwright install
+    - name: Run tests
+      env:
+        TOOLIUM_DRIVER_HEADLESS: Driver_headless=true
+      run: |
+        behave web_playwright_behave/features --junit --junit-directory web_playwright_behave/output/reports/
+      continue-on-error: true
+    - name: Upload output folder
+      uses: actions/upload-artifact@v7
+      if: always()
+      with:
+        name: playwright-tests-output-${{ matrix.python-version }}
+        path: web_playwright_behave/output
+    - name: Publish test results
+      uses: dorny/test-reporter@v3
+      if: always()
+      with:
+        name: playwright tests results (${{ matrix.python-version }})
+        path: web_playwright_behave/output/reports/*.xml
+        reporter: java-junit
+        fail-on-error: true
+
+  combine-output-artifacts:
+    runs-on: ubuntu-latest
+    needs: [behave-tests, pytest-tests, nose2-tests, playwright-tests]
+    if: always()
+    steps:
+    - name: Download all artifacts
+      uses: actions/download-artifact@v5
+      with:
+        path: all-artifacts
+    - name: Create combined structure
+      run: |
+        mkdir -p tests-output
+        echo "=== Downloaded artifacts ==="
+        ls -la all-artifacts/
+        # Organize by framework and Python version
+        for framework in behave pytest nose2 playwright; do
+          mkdir -p "tests-output/$framework"
+          for version in 3.10 3.11 3.12 3.13; do
+            if [ -d "all-artifacts/${framework}-tests-output-${version}" ]; then
+              echo "Moving $framework Python $version output..."
+              mv "all-artifacts/${framework}-tests-output-${version}" "tests-output/$framework/python-${version}"
+            fi
+          done
+        done
+        echo "=== Combined structure ==="
+        tree tests-output || find tests-output -type d
+        echo "=== Total files ==="
+        find tests-output -type f | wc -l
+    - name: Upload combined output artifacts
+      uses: actions/upload-artifact@v7
+      with:
+        name: tests-output
+        path: tests-output/

web_nose2/tests/test_web_visual_testing.py

@@ -87,3 +87,6 @@ def test_successful_login_logout_visualtesting_examples(self):
 
         # Assert the full screen
         self.assert_full_screenshot('login_logout')
+
+    # Add local attribute to allow skiping it in CI, it can only be executed locally
+    test_successful_login_logout_visualtesting_examples.local = 1