Add automated testing infrastructure (Vitest + Playwright E2E) #110

Merged
litruv merged 12 commits from issue-109-automated-testing into main 2026-11-03 13:51:15 +11:00
9 changed files with 1458 additions and 6 deletions
+94
View File
@@ -0,0 +1,94 @@
name: Tests
on:
pull_request:
jobs:
unit:
# act_runner mounts /var/run/docker.sock into every container job, so a fork PR reaches
# the Docker daemon on the runner host. Both jobs need the same guard.
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '22'
- name: Install dependencies
run: npm ci
- name: Run unit tests
run: npm run test:unit
e2e:
# A fork PR controls both the test files and these steps, so only same-repository
# PRs can be trusted with the E2E_PASSWORD secret.
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
# Needs the DOCKER-USER exception on the runner host: ruv.wtf is a macvlan interface on a
# container there, so the reply to a bridge-networked job arrives on that container's bridge
# and Docker's inter-bridge isolation drops it. The exception is host-wide, so it also covers
# fork PR jobs, and the bridge name is Docker-generated: recreating the caddy container
# renames it and the test fails closed.
env:
# Dedicated test account on the homeserver, not a personal one.
E2E_HOMESERVER: ruv.wtf
E2E_USER: giteaci
# Repo secret: E2E_PASSWORD. The test fails with a clear message when it is missing.
E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }}
steps:
# .gitmodules uses a relative URL, so the runner resolves it against origin.
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: true
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '22'
- name: Install dependencies
run: npm ci
# The E2E run drives the vite dev server, so cinny's dependencies are needed too.
- name: Install dependencies (cinny)
working-directory: ./cinny
run: npm ci
# Electron needs system libraries the runner image does not ship. Do not narrow this to
# chromium: Playwright's chromium dep set covers its own bundled browser, not Electron,
# and drops libgtk-3-0t64, libegl1, libegl-mesa0, libgles2, libsecret-1-0, libxtst6 and
# libxss1, which Electron needs to launch. Narrowing it fails with "Process failed to
# launch!".
- name: Install system libraries for Electron
run: npx playwright install-deps
# The container runner image has no xvfb and no fresh apt lists. The host runner already
# has xvfb, so the guard skips the install there; both share the ubuntu-latest label, so
# which one runs is not fixed.
- name: Ensure a display for Electron
run: command -v xvfb-run >/dev/null || (sudo apt-get update && sudo apt-get install -y xvfb)
# The login form only renders after the auth-flows request succeeds, so an
# unreachable homeserver shows up as a 30s locator timeout. Fail fast instead, on
# connect failure or a non-2xx status.
- name: Homeserver reachable from the runner
run: curl -sSf --max-time 20 -o /dev/null -w '%{http_code}\n' "https://$E2E_HOMESERVER/.well-known/matrix/client"
- name: Run login E2E
run: xvfb-run -a npm run test:e2e
# Gitea implements the v3 artifact protocol; v4 rejects it as GHES.
- name: Upload test artifacts
if: failure()
uses: actions/upload-artifact@v3
with:
name: e2e-artifacts
path: |
test-results
playwright-report
+2
View File
@@ -5,3 +5,5 @@ devAssets
.DS_Store
dist-electron/
playwright-report/
test-results/
+22
View File
@@ -30,6 +30,28 @@ To build the app locally, run:
To start local dev server, run:
* `npm run dev`
## Testing
* `npm run test` - unit and E2E
* `npm run test:unit` - Vitest
* `npm run test:e2e` - Playwright driving Electron
The E2E test drives the real Electron app against the vite dev server, so it needs the
cinny dependencies installed and an X display. On a headless machine run it under xvfb:
`xvfb-run -a npm run test:e2e`.
The E2E test logs in to a live homeserver over the network, so it needs network access and a
working test account. It uses a dedicated account, never a personal one:
Variable | Default
---|---
`E2E_HOMESERVER` | `ruv.wtf`
`E2E_USER` | `giteaci`
`E2E_PASSWORD` | required, read from the environment, never committed
A failed E2E run attaches a screenshot and the main and renderer console logs to the
report. CI uploads `test-results/` and `playwright-report/` as artifacts.
## Plugin System
Paarrot includes a plugin system for extending and customising the app with JavaScript modules.
+1120 -5
View File
File diff suppressed because it is too large Load Diff
+6 -1
View File
@@ -20,7 +20,10 @@
"build:linux": "npm run build -- --linux",
"build:win": "npm run build -- --win",
"postman:generate": "node scripts/generate-postman-collection.js",
"generate:update-manifest": "node scripts/generate-update-manifest.mjs"
"generate:update-manifest": "node scripts/generate-update-manifest.mjs",
"test": "npm run test:unit && npm run test:e2e",
"test:unit": "vitest run",
"test:e2e": "playwright test"
},
"keywords": [],
"author": {
@@ -40,6 +43,7 @@
"devDependencies": {
"@actions/github": "9.1.1",
"@electron/rebuild": "4.2.0",
"@playwright/test": "1.64.1",
"concurrently": "10.0.3",
"cross-env": "10.1.0",
"electron": "43.1.0",
@@ -47,6 +51,7 @@
"node-fetch": "3.3.2",
"png2icons": "2.0.1",
"sharp": "0.35.3",
"vitest": "5.1.0",
"wait-on": "9.0.10"
},
"allowScripts": {
+26
View File
@@ -0,0 +1,26 @@
import { defineConfig } from '@playwright/test';
export default defineConfig({
testDir: './tests/e2e',
// Electron launch plus first paint is far slower than a page load.
timeout: 120_000,
expect: { timeout: 30_000 },
// One Electron process at a time: the app takes a single-instance lock and
// every test shares the same vite dev server.
workers: 1,
// The suite logs in to a live homeserver, where Synapse login rate limiting is a real flake.
retries: process.env.CI ? 1 : 0,
// An explicit reporter replaces the default, so list stays for the CI log.
reporter: [['list'], ['html']],
// A trace captures the login POST body, which contains the password. Pin it off rather than
// relying on the version default.
trace: 'off',
// cinny/vite.config.js binds 0.0.0.0 with fs.allow: ['..'], which is exposed on a shared
// CI runner, so pin the dev server to loopback. The flag has to reach vite itself:
// `npm run dev:vite -- --host` lands on sh's positional params, not on vite.
// localhost is also the URL electron/main.js loads, so the page keeps that origin.
webServer: {
command: 'sh -c "cd cinny && npm start -- --host localhost"',
url: 'http://localhost:38347',
},
});
+147
View File
@@ -0,0 +1,147 @@
import { test, expect, _electron } from '@playwright/test';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
const HOMESERVER = process.env.E2E_HOMESERVER ?? 'ruv.wtf';
const USER = process.env.E2E_USER ?? 'giteaci';
const PASSWORD = process.env.E2E_PASSWORD;
let app;
let page;
let logs = [];
let profile;
let crashed = false;
// ServerPicker gives its input no name or test id, so position is the only handle on it;
// this breaks if the form ever gains a field before the ones below.
const serverInput = () => page.locator('input').first();
const usernameInput = () => page.locator('input[name="usernameInput"]');
const passwordInput = () => page.locator('input[name="passwordInput"]');
const submitButton = () => page.locator('button[type="submit"]');
async function expectLoginForm() {
await expect(serverInput()).toBeVisible();
await expect(usernameInput()).toBeVisible();
await expect(passwordInput()).toBeVisible();
await expect(submitButton()).toBeVisible();
}
async function chooseServer(server) {
await serverInput().fill(server);
// The server field commits on Enter, which re-runs discovery and reloads the auth flows.
await serverInput().press('Enter');
// fill() does not prove the value stuck; a cancelled debounce would keep matrix.org.
await expect(serverInput()).toHaveValue(server);
await expectLoginForm();
}
async function submitLogin(username, password) {
await usernameInput().fill(username);
await passwordInput().fill(password);
await submitButton().click();
}
test.beforeEach(async () => {
logs = [];
crashed = false;
if (!PASSWORD) {
throw new Error('E2E_PASSWORD is not set. Export it, or add it as the CI secret.');
}
// Fresh profile per run. electron/main.js takes a single-instance lock keyed to
// the user data dir, so a shared profile collides with a running Paarrot.
profile = fs.mkdtempSync(path.join(os.tmpdir(), 'paarrot-e2e-'));
app = await _electron.launch({ args: ['.', `--user-data-dir=${profile}`] });
app.on('console', (msg) => logs.push(`[main] ${msg.type()}: ${msg.text()}`));
page = await app.firstWindow();
// main.js opens DevTools in dev mode, and that window can be the first one registered.
while (page.url().startsWith('devtools://')) {
// Recorded so a hang here names which windows existed instead of reporting a bare timeout.
logs.push(`[window] ${page.url()}`);
page = app.windows().find((w) => !w.url().startsWith('devtools://'))
?? await new Promise((resolve) => app.once('window', resolve));
}
page.on('console', (msg) => logs.push(`[renderer] ${msg.type()}: ${msg.text()}`));
page.on('pageerror', (err) => logs.push(`[renderer] pageerror: ${err}`));
// crash is render-process-gone, so it does not fire on the MatrixError or the ServiceWorker error.
page.on('crash', () => {
crashed = true;
logs.push('[renderer] page crashed');
});
await chooseServer(HOMESERVER);
});
// Config-based screenshots do not fire for a page taken from app.firstWindow(), and console
// capture is not covered by config at all, so this hook stays manual.
test.afterEach(async ({}, testInfo) => {
if (testInfo.status !== testInfo.expectedStatus) {
// Console first: a failed launch has no app and no page, so this is the only artifact that
// path can produce, and it is empty by construction because no handler is attached before
// launch — the failure message in the report is the diagnosis there. A crashed page produces
// no screenshot; the crash line in the console names it.
await testInfo.attach('console.log', { body: logs.join('\n'), contentType: 'text/plain' });
if (page && !page.isClosed()) {
// screenshot() throws on a crashed page, which would mask the original failure.
const shot = await page.screenshot().catch(() => null);
if (shot) await testInfo.attach('failure.png', { body: shot, contentType: 'image/png' });
}
}
if (app) {
await app
.close()
// main.js preventDefault()s window close to minimise to tray, so close can hang.
.catch(() => app.process().kill('SIGTERM'));
}
// Reachable on launch failure, so the temp profile does not leak.
if (profile) fs.rmSync(profile, { recursive: true, force: true });
});
async function expectAuthenticated() {
// Login complete reloads into the client shell, which lands on the last active path
// (state/lastActivePath.ts) or the inbox, so assert a shell route rather than any URL.
await expect(page).toHaveURL(/#\/(inbox|home|room|space)/);
await expect
.poll(() => page.evaluate(() => JSON.parse(localStorage.getItem('currentSessionUserId') ?? 'null')))
.toBe(`@${USER}:${HOMESERVER}`);
// A truthy accessToken only proves a string exists, so prove it authenticates. Only the
// status is asserted, so a failure cannot put a live token into the CI log or the report.
await expect
.poll(async () => {
const token = await page.evaluate(() => JSON.parse(localStorage.getItem('matrixSessions') ?? '[]')[0]?.accessToken);
if (!token) return 0;
// /versions and /account 404 on ruv.wtf; /capabilities exists and requires a valid token.
const res = await fetch(`https://${HOMESERVER}/_matrix/client/v3/capabilities`, { headers: { Authorization: `Bearer ${token}` } });
return res.status;
})
.toBe(200);
}
test('test account can log in and reaches the authenticated app', async () => {
await submitLogin(USER, PASSWORD);
await expectAuthenticated();
});
test('failed login reports an error and keeps the renderer alive', async () => {
await submitLogin(USER, 'not-the-password');
// A rate-limited run renders LoginError.RateLimited instead of the wrong-password message,
// so accept both branches rather than asserting one exact string.
await expect(page.getByText(/Invalid Username or Password\.|rate-limited by server/)).toBeVisible();
// Asserted before the form check: on a crashed page the locator throws TargetClosedError,
// which names the wrong thing. Pageerrors go into the artifact but are not asserted — the
// failed login itself produces MatrixError, and dev mode produces a ServiceWorker registration
// error. No second login here: every run costs Synapse rc_login budget, and the re-login was
// the main flake source.
await expect(crashed).toBe(false);
await expect(usernameInput()).toBeVisible();
});
+33
View File
@@ -0,0 +1,33 @@
import { afterAll, beforeAll, expect, test } from 'vitest';
import PaarrotAPIServer from '../../electron/api-server.js';
const api = new PaarrotAPIServer(null);
let base;
beforeAll(async () => {
// Port 0 lets the OS pick a free port, so the test never collides with a running Paarrot.
await api.start(0);
base = `http://127.0.0.1:${api.server.address().port}`;
});
afterAll(async () => {
await api.stop();
});
test('GET /health reports the API server is up', async () => {
const res = await fetch(`${base}/health`);
expect(res.status).toBe(200);
expect(await res.json()).toEqual({ status: 'ok', app: 'Paarrot API', version: '1.0.0' });
});
test('unknown endpoint answers 404 with the endpoint list', async () => {
const res = await fetch(`${base}/not-an-endpoint`);
expect(res.status).toBe(404);
const body = await res.json();
expect(body.error).toBe('Endpoint not found');
expect(body.availableEndpoints).toContain('GET /health');
});
+8
View File
@@ -0,0 +1,8 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
// The default pattern would also pick up the Electron spec in tests/e2e.
include: ['tests/unit/**/*.test.js'],
},
});