diff --git a/.github/pie-behaviour-tests/Dockerfile b/.github/pie-behaviour-tests/Dockerfile
new file mode 100644
index 0000000..d038dd1
--- /dev/null
+++ b/.github/pie-behaviour-tests/Dockerfile
@@ -0,0 +1,43 @@
+# An approximately reproducible, but primarily isolated, environment for
+# running the acceptance tests:
+#
+# docker buildx build --file .github/pie-behaviour-tests/Dockerfile -t pie-behat-test .
+# docker run --volume .:/github/workspace -ti pie-behat-test
+FROM ubuntu:24.04
+
+# Add the `unzip` package which PIE uses to extract .zip files
+RUN export DEBIAN_FRONTEND="noninteractive"; \
+    set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+      unzip curl jq wget g++ gcc make autoconf libtool bison re2c pkg-config \
+      ca-certificates libxml2-dev libssl-dev; \
+    update-ca-certificates ; \
+    rm -rf /var/lib/apt/lists/*
+
+# Compile PHP
+ARG PHP_VERSION=8.4
+RUN mkdir -p /usr/local/src/php; \
+    cd /usr/local/src/php; \
+    FULL_LATEST_VERSION=`curl -fsSL "https://www.php.net/releases/index.php?json&max=1&version=$PHP_VERSION" | jq -r '.[].source[]|select(.filename|endswith(".gz")).filename'`; \
+    wget -O php.tgz "https://www.php.net/distributions/$FULL_LATEST_VERSION" ; \
+    tar zxf php.tgz ; \
+    rm php.tgz ; \
+    ls -l ; \
+    cd * ; \
+    ls -l ; \
+    ./buildconf --force ; \
+    ./configure --without-sqlite3 --disable-pdo --disable-dom --disable-xml --disable-xmlreader --disable-xmlwriter --disable-json --with-openssl ; \
+    make -j$(nproc) ; \
+    make install
+
+RUN touch /usr/local/lib/php.ini
+
+COPY --from=ghcr.io/php/pie:bin /pie /usr/bin/pie
+RUN pie install xdebug/xdebug
+
+WORKDIR /github/workspace
+
+ENV USING_PIE_BEHAT_DOCKERFILE=1
+ENTRYPOINT ["php", "vendor/bin/behat"]
+CMD ["--no-snippets"]
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
index c877fac..acff087 100644
--- a/.github/workflows/continuous-integration.yml
+++ b/.github/workflows/continuous-integration.yml
@@ -138,38 +138,18 @@ jobs:
       matrix:
         operating-system:
           - ubuntu-latest
-          - windows-latest
         php-versions:
           - '8.1'
           - '8.2'
           - '8.3'
           - '8.4'
-          - '8.5'
     steps:
-      - name: Setup PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          php-version: ${{ matrix.php-versions }}
-          extensions: none, intl, zip
-          tools: composer
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v5
-      - name: Testing
-        run: |
-          php -m
-          bin/pie show --all
-          which composer
-          composer -v
       - uses: ramsey/composer-install@v3
-      - name: Run Behat on Windows
-        if: matrix.operating-system == 'windows-latest'
-        run: vendor/bin/behat --no-snippets --tags="~@non-windows"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run Behat on non-Windows
-        if: matrix.operating-system != 'windows-latest'
-        run: sudo vendor/bin/behat --no-snippets
+      - name: Build
+        run: docker buildx build --file .github/pie-behaviour-tests/Dockerfile --build-arg PHP_VERSION=${{ matrix.php-versions }} -t pie-behat-test .
+      - name: Run Behat
+        run: docker run --volume .:/github/workspace pie-behat-test
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
diff --git a/behat.php b/behat.php
index 8202fe0..c5fa1a5 100644
--- a/behat.php
+++ b/behat.php
@@ -8,6 +8,25 @@ use Behat\Config\Profile;
 use Behat\Config\Suite;
 use Php\PieBehaviourTest\CliContext;
 
+if (getenv('USING_PIE_BEHAT_DOCKERFILE') !== '1') {
+    echo <<<'HELP'
+⚠️ ⚠️ ⚠️  STOP! ⚠️ ⚠️ ⚠️
+
+This test suite tinkers with your system, and has lots of expectations about
+the system it is running on, so we HIGHLY recommend you run it using the
+provided Dockerfile:
+
+  docker buildx build --file .github/actions/pie-behaviour-tests/Dockerfile -t pie-behat-test .
+  docker run --volume .:/github/workspace -ti pie-behat-test
+
+If you are really sure, and accept that the test suite installs/uninstalls
+stuff from your system, and might break your stuff, set
+USING_PIE_BEHAT_DOCKERFILE=1 in your environment.
+
+HELP;
+    exit(1);
+}
+
 $profile = (new Profile('default'))
     ->withSuite(
         (new Suite('default'))