Add visual regression tests (#994)

Co-authored-by: Roman Pronskiy <roman@pronskiy.com>
Co-authored-by: Derick Rethans <derick@php.net>

.github/workflows/pr-closed.yml

@@ -13,4 +13,6 @@ jobs:
           host: ${{ secrets.PREVIEW_REMOTE_HOST }}
           username: ${{ secrets.PREVIEW_REMOTE_USER }}
           key: ${{ secrets.PREVIEW_SSH_KEY }}
-          script: bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
+          script: |
+            bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
+            bash /home/thephpfoundation/scripts/pr_closed.sh web-php-regression-report ${{ github.event.number }}

.github/workflows/pr-preview.yml

@@ -32,4 +32,99 @@ jobs:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           edit-mode: 'replace'
           body: |
-            🚀 Commit ${{ github.sha }} Deployed on https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
+            🚀 Preview for commit ${{ github.sha }} can be found at https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
+
+  tests_visual:
+    name: "Visual Tests"
+
+    runs-on: "ubuntu-latest"
+    if: github.repository_owner == 'php'
+
+    strategy:
+      matrix:
+        php-version:
+          - "8.2"
+        node-version:
+          - "22.x"
+
+    env:
+      HTTP_HOST: "localhost:8080"
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: "Set up PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
+          php-version: "${{ matrix.php-version }}"
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: "Set up problem matchers for PHP"
+        run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
+
+      - name: "Set up problem matchers for phpunit/phpunit"
+        run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
+
+      - name: "Determine composer cache directory"
+        run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
+
+      - name: "Cache dependencies installed with composer"
+        uses: "actions/cache@v3"
+        with:
+          path: "${{ env.COMPOSER_CACHE_DIR }}"
+          key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
+          restore-keys: "php-${{ matrix.php-version }}-composer-"
+
+      - name: "Install dependencies with composer"
+        run: "composer install --ansi --no-interaction --no-progress"
+
+      - name: "Install dependencies"
+        run: "yarn install"
+
+      - name: "Install Playwright"
+        run: "npx playwright install"
+
+      - name: "Run visual tests"
+        run: "make tests_visual"
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+      - uses: easingthemes/ssh-deploy@main
+        if: ${{ !cancelled() }}
+        with:
+          REMOTE_HOST: ${{ secrets.PREVIEW_REMOTE_HOST }}
+          REMOTE_USER: ${{ secrets.PREVIEW_REMOTE_USER }}
+          SSH_PRIVATE_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+          SOURCE: "playwright-report/"
+          TARGET: "/home/thephpfoundation/preview/web-php-regression-report-pr-${{ github.event.number }}/public"
+          SCRIPT_BEFORE: bash /home/thephpfoundation/scripts/pr_created_pre.sh web-php-regression-report ${{ github.event.number }}
+
+      - uses: peter-evans/find-comment@v3
+        if: ${{ !cancelled() }}
+        id: snapshot
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: 'github-actions[bot]'
+
+      - uses: peter-evans/create-or-update-comment@v4
+        if: ${{ !cancelled() }}
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-id: ${{ steps.snapshot.outputs.comment-id }}
+          edit-mode: 'replace'
+          body: |
+            🚀 Regression report for commit ${{ github.sha }} is at https://web-php-regression-report-pr-${{ github.event.number }}.preview.thephp.foundation

.github/workflows/update-screenshots.yaml

@@ -0,0 +1,74 @@
+# https://docs.github.com/en/actions
+
+name: "Update screenshots"
+
+on:
+  workflow_dispatch:
+
+jobs:
+  tests_update_snapshots:
+    name: "Update Tests snapshots"
+
+    runs-on: "ubuntu-latest"
+
+    strategy:
+      matrix:
+        php-version:
+          - "8.2"
+        node-version:
+          - "22.x"
+
+    env:
+      HTTP_HOST: "localhost:8080"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+
+      - name: "Set up PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
+          php-version: "${{ matrix.php-version }}"
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: "Set up problem matchers for PHP"
+        run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
+
+      - name: "Set up problem matchers for phpunit/phpunit"
+        run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
+
+      - name: "Determine composer cache directory"
+        run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
+
+      - name: "Cache dependencies installed with composer"
+        uses: "actions/cache@v3"
+        with:
+          path: "${{ env.COMPOSER_CACHE_DIR }}"
+          key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
+          restore-keys: "php-${{ matrix.php-version }}-composer-"
+
+      - name: "Install dependencies with composer"
+        run: "composer install --ansi --no-interaction --no-progress"
+
+      - name: "Install dependencies"
+        run: "yarn install"
+
+      - name: "Install Playwright"
+        run: "npx playwright install"
+
+      - name: "Run visual tests"
+        run: "make tests_update_snapshots"
+
+      - name: Update snapshots
+        uses: test-room-7/action-update-file@v1
+        if: ${{ !cancelled() }}
+        with:
+          file-path: tests/Visual/**/*
+          commit-msg: Update snapshots
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -5,3 +5,8 @@ backend/mirror.gif
 backend/mirror.png
 backend/mirror.jpg
 backend/GeoIP.dat
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

.htaccess

@@ -1,4 +1,4 @@
-<FilesMatch "\.(inc|sql)$">
+<FilesMatch "\.(inc|sql|lock)$">
   Order allow,deny
   Deny from all
 </FilesMatch>

Makefile

@@ -1,6 +1,7 @@
 .EXPORT_ALL_VARIABLES:
 
 HTTP_HOST:=localhost:8080
+CORES?=$(shell (nproc  || sysctl -n hw.ncpu) 2> /dev/null)
 
 .PHONY: it
 it: coding-standards tests ## Runs all the targets
@@ -21,7 +22,19 @@ help: ## Displays this list of targets with descriptions
 tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit
 	vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=unit
 	rm -rf tests/server.log
-	tests/server start; vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop
+	tests/server start;
+	vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end;
+	tests/server stop
+
+tests_visual:
+	tests/server start;
+	npx playwright test --workers=$(CORES)
+	tests/server stop
+
+tests_update_snapshots:
+	tests/server start;
+	npx playwright test --update-snapshots
+	tests/server stop
 
 vendor: composer.json composer.lock
 	composer validate --strict

package.json

@@ -0,0 +1,8 @@
+{
+  "name": "php.net",
+  "devDependencies": {
+    "@playwright/test": "^1.44.0",
+    "@types/node": "^20.12.11"
+  },
+  "scripts": {}
+}

playwright.config.ts

@@ -0,0 +1,34 @@
+import {defineConfig, devices} from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+    testDir: './tests/Visual',
+    /* Run tests in files in parallel */
+    fullyParallel: true,
+    /* Fail the build on CI if you accidentally left test.only in the source code. */
+    forbidOnly: !!process.env.CI,
+    /* Retry on CI only */
+    retries: process.env.CI ? 2 : 0,
+    /* Opt out of parallel tests on CI. */
+    workers: process.env.CI ? 1 : undefined,
+    /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+    reporter: 'html',
+    /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+    use: {
+        /* Base URL to use in actions like `await page.goto('/')`. */
+        // baseURL: 'http://127.0.0.1:3000',
+
+        /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+        trace: 'on-first-retry',
+    },
+    timeout: 0,
+
+    projects: [
+        {
+            name: 'chromium',
+            use: {...devices['Desktop Chrome']},
+        },
+    ],
+});

tests/Visual/SmokeTest.spec.ts

@@ -0,0 +1,63 @@
+import {expect, test} from '@playwright/test';
+
+export type TestPageOptions = {
+    path: string
+    options?: object
+    evaluate?: () => any
+}
+
+const items: TestPageOptions[] = [
+    {
+        path: 'index.php',
+        evaluate: () => {
+            const selector = document.querySelector('.elephpants');
+            selector.remove()
+        }
+    },
+    {
+        path: 'archive/1998.php',
+        evaluate: () => {
+            const selector = document.querySelector('.elephpants');
+            selector.remove()
+        }
+    },
+    {path: 'releases/8_3_6.php'},
+    {path: 'releases/8.0/index.php'},
+    {path: 'releases/8.1/index.php'},
+    {path: 'releases/8.2/index.php'},
+    {path: 'releases/8.3/index.php'},
+    {path: 'manual/index.php'},
+    {path: 'manual/php5.php'},
+    {
+        path: 'conferences/index.php',
+        options: {
+            fullPage: false,
+        }
+    },
+]
+
+for (const item of items) {
+    test(`testing with ${item.path}`, async ({page}, testInfo) => {
+        testInfo.snapshotSuffix = '';
+
+        const httpHost = process.env.HTTP_HOST
+
+        if (typeof httpHost !== 'string') {
+            throw new Error('Environment variable "HTTP_HOST" is not set.')
+        }
+
+        await page.goto(`${httpHost}/${item.path}`)
+
+        if (typeof item.evaluate === 'function') {
+            await page.evaluate(item.evaluate)
+        }
+
+        await expect(page).toHaveScreenshot(
+            `tests/screenshots/${item.path}.png`,
+            item.options ?? {
+                fullPage: true,
+                timeout: 10000,
+            }
+        )
+    })
+}

yarn.lock

@@ -0,0 +1,41 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@playwright/test@^1.44.0":
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.0.tgz#ac7a764b5ee6a80558bdc0fcbc525fcb81f83465"
+  integrity sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg==
+  dependencies:
+    playwright "1.44.0"
+
+"@types/node@^20.12.11":
+  version "20.12.11"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.11.tgz#c4ef00d3507000d17690643278a60dc55a9dc9be"
+  integrity sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==
+  dependencies:
+    undici-types "~5.26.4"
+
+fsevents@2.3.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
+playwright-core@1.44.0:
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.0.tgz#316c4f0bca0551ffb88b6eb1c97bc0d2d861b0d5"
+  integrity sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==
+
+playwright@1.44.0:
+  version "1.44.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.0.tgz#22894e9b69087f6beb639249323d80fe2b5087ff"
+  integrity sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==
+  dependencies:
+    playwright-core "1.44.0"
+  optionalDependencies:
+    fsevents "2.3.2"
+
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==