Apply the #109 review panel: fork guard, loopback dev server, real authenticated assertions.
Playwright's webServer replaces the custom globalSetup. The e2e job is gated to same-repo PRs because a fork PR controls both the test files and the workflow steps while the E2E_PASSWORD secret is in the environment. The dev server is pinned to loopback, the login screen is asserted before filling, the authenticated state is proven by a token-authenticated request, the token value is never printed, the failed-login test no longer asserts a tautological exitCode, and the DevTools window is no longer mistaken for the app window.
This commit is contained in:
@@ -22,6 +22,9 @@ jobs:
|
|||||||
run: npm run test:unit
|
run: npm run test:unit
|
||||||
|
|
||||||
e2e:
|
e2e:
|
||||||
|
# A fork PR controls both the test files and these steps, so only same-repository
|
||||||
|
# PRs can be trusted with the E2E_PASSWORD secret.
|
||||||
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
# Dedicated test account on the homeserver, not a personal one.
|
# Dedicated test account on the homeserver, not a personal one.
|
||||||
@@ -30,33 +33,16 @@ jobs:
|
|||||||
# Repo secret: E2E_PASSWORD. The test fails with a clear message when it is missing.
|
# Repo secret: E2E_PASSWORD. The test fails with a clear message when it is missing.
|
||||||
E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }}
|
E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }}
|
||||||
steps:
|
steps:
|
||||||
|
# .gitmodules uses a relative URL, so the runner resolves it against origin.
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
submodules: false
|
submodules: true
|
||||||
|
|
||||||
- name: Checkout submodules manually
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
# Use the same host/scheme the runner already used for origin (not public HTTPS).
|
|
||||||
ORIGIN_URL=$(git remote get-url origin)
|
|
||||||
ORIGIN_URL=${ORIGIN_URL%/}
|
|
||||||
ORIGIN_URL=${ORIGIN_URL%.git}
|
|
||||||
BASE_URL=${ORIGIN_URL%/*}
|
|
||||||
CINNY_URL="${BASE_URL}/cinny.git"
|
|
||||||
echo "Rewriting submodule URL to ${CINNY_URL}"
|
|
||||||
git config submodule.cinny.url "${CINNY_URL}"
|
|
||||||
git submodule sync --recursive
|
|
||||||
git submodule update --init --recursive
|
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
|
||||||
cache-dependency-path: |
|
|
||||||
package-lock.json
|
|
||||||
cinny/package-lock.json
|
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
|
|||||||
@@ -40,7 +40,8 @@ The E2E test drives the real Electron app against the vite dev server, so it nee
|
|||||||
cinny dependencies installed and an X display. On a headless machine run it under xvfb:
|
cinny dependencies installed and an X display. On a headless machine run it under xvfb:
|
||||||
`xvfb-run -a npm run test:e2e`.
|
`xvfb-run -a npm run test:e2e`.
|
||||||
|
|
||||||
The login test uses a dedicated test account, never a personal one:
|
The E2E test logs in to a live homeserver over the network, so it needs network access and a
|
||||||
|
working test account. It uses a dedicated account, never a personal one:
|
||||||
|
|
||||||
Variable | Default
|
Variable | Default
|
||||||
---|---
|
---|---
|
||||||
|
|||||||
+12
-3
@@ -4,10 +4,19 @@ export default defineConfig({
|
|||||||
testDir: './tests/e2e',
|
testDir: './tests/e2e',
|
||||||
// Electron launch plus first paint is far slower than a page load.
|
// Electron launch plus first paint is far slower than a page load.
|
||||||
timeout: 120_000,
|
timeout: 120_000,
|
||||||
|
expect: { timeout: 30_000 },
|
||||||
// One Electron process at a time: the app takes a single-instance lock and
|
// One Electron process at a time: the app takes a single-instance lock and
|
||||||
// every test shares the same vite dev server.
|
// every test shares the same vite dev server.
|
||||||
workers: 1,
|
workers: 1,
|
||||||
outputDir: 'test-results',
|
// The suite logs in to a live homeserver, where Synapse login rate limiting is a real flake.
|
||||||
reporter: [['list'], ['html', { open: false }]],
|
retries: process.env.CI ? 1 : 0,
|
||||||
globalSetup: './tests/e2e/dev-server.mjs',
|
reporter: [['list'], ['html']],
|
||||||
|
// cinny/vite.config.js binds 0.0.0.0 with fs.allow: ['..'], which is exposed on a shared
|
||||||
|
// CI runner, so pin the dev server to loopback. The flag has to reach vite itself:
|
||||||
|
// `npm run dev:vite -- --host` lands on sh's positional params, not on vite.
|
||||||
|
// localhost is also the URL electron/main.js loads, so the page keeps that origin.
|
||||||
|
webServer: {
|
||||||
|
command: 'sh -c "cd cinny && npm start -- --host localhost"',
|
||||||
|
url: 'http://localhost:38347',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
import { spawn } from 'node:child_process';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
|
|
||||||
// electron/main.js loads this URL whenever the app is not packaged, so the E2E
|
|
||||||
// run needs the vite dev server up. It is the same server `npm run dev` uses.
|
|
||||||
export const DEV_URL = 'http://localhost:38347';
|
|
||||||
|
|
||||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
|
||||||
|
|
||||||
const reachable = async () => {
|
|
||||||
try {
|
|
||||||
const res = await fetch(DEV_URL, { signal: AbortSignal.timeout(3000) });
|
|
||||||
return res.ok;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export default async function globalSetup() {
|
|
||||||
// A dev server the developer already started is reused rather than restarted.
|
|
||||||
if (await reachable()) return () => {};
|
|
||||||
|
|
||||||
const child = spawn('npm', ['run', 'dev:vite'], { cwd: ROOT, detached: true, stdio: 'inherit' });
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt < 60; attempt++) {
|
|
||||||
if (await reachable()) {
|
|
||||||
// detached puts npm and its sh -c child in their own process group.
|
|
||||||
return () => process.kill(-child.pid, 'SIGTERM');
|
|
||||||
}
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
||||||
}
|
|
||||||
|
|
||||||
process.kill(-child.pid, 'SIGTERM');
|
|
||||||
throw new Error(`Vite dev server did not come up at ${DEV_URL}`);
|
|
||||||
}
|
|
||||||
+49
-22
@@ -9,47 +9,66 @@ const PASSWORD = process.env.E2E_PASSWORD;
|
|||||||
|
|
||||||
let app;
|
let app;
|
||||||
let page;
|
let page;
|
||||||
let logs;
|
let logs = [];
|
||||||
|
let profile;
|
||||||
|
|
||||||
|
// ServerPicker gives its input no name or test id, so position is the only handle on it;
|
||||||
|
// this breaks if the form ever gains a field before the ones below.
|
||||||
const serverInput = () => page.locator('input').first();
|
const serverInput = () => page.locator('input').first();
|
||||||
const usernameInput = () => page.locator('input[name="usernameInput"]');
|
const usernameInput = () => page.locator('input[name="usernameInput"]');
|
||||||
const passwordInput = () => page.locator('input[name="passwordInput"]');
|
const passwordInput = () => page.locator('input[name="passwordInput"]');
|
||||||
|
const submitButton = () => page.locator('button[type="submit"]');
|
||||||
|
|
||||||
|
async function expectLoginForm() {
|
||||||
|
await expect(serverInput()).toBeVisible();
|
||||||
|
await expect(usernameInput()).toBeVisible();
|
||||||
|
await expect(passwordInput()).toBeVisible();
|
||||||
|
await expect(submitButton()).toBeVisible();
|
||||||
|
}
|
||||||
|
|
||||||
async function chooseServer(server) {
|
async function chooseServer(server) {
|
||||||
await serverInput().fill(server);
|
await serverInput().fill(server);
|
||||||
// The server field commits on Enter, which re-runs discovery and reloads the auth flows.
|
// The server field commits on Enter, which re-runs discovery and reloads the auth flows.
|
||||||
await serverInput().press('Enter');
|
await serverInput().press('Enter');
|
||||||
await expect(usernameInput()).toBeVisible();
|
await expectLoginForm();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function submitLogin(username, password) {
|
async function submitLogin(username, password) {
|
||||||
await usernameInput().fill(username);
|
await usernameInput().fill(username);
|
||||||
await passwordInput().fill(password);
|
await passwordInput().fill(password);
|
||||||
await page.locator('button[type="submit"]').click();
|
await submitButton().click();
|
||||||
}
|
}
|
||||||
|
|
||||||
test.beforeEach(async () => {
|
test.beforeEach(async () => {
|
||||||
|
logs = [];
|
||||||
|
|
||||||
if (!PASSWORD) {
|
if (!PASSWORD) {
|
||||||
throw new Error('E2E_PASSWORD is not set. Export it, or add it as the CI secret.');
|
throw new Error('E2E_PASSWORD is not set. Export it, or add it as the CI secret.');
|
||||||
}
|
}
|
||||||
|
|
||||||
logs = [];
|
|
||||||
|
|
||||||
// Fresh profile per run. electron/main.js takes a single-instance lock keyed to
|
// Fresh profile per run. electron/main.js takes a single-instance lock keyed to
|
||||||
// the user data dir, so a shared profile collides with a running Paarrot.
|
// the user data dir, so a shared profile collides with a running Paarrot.
|
||||||
const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'paarrot-e2e-'));
|
profile = fs.mkdtempSync(path.join(os.tmpdir(), 'paarrot-e2e-'));
|
||||||
app = await _electron.launch({ args: ['.', `--user-data-dir=${profile}`] });
|
app = await _electron.launch({ args: ['.', `--user-data-dir=${profile}`] });
|
||||||
|
|
||||||
app.on('console', (msg) => logs.push(`[main] ${msg.type()}: ${msg.text()}`));
|
app.on('console', (msg) => logs.push(`[main] ${msg.type()}: ${msg.text()}`));
|
||||||
|
|
||||||
page = await app.firstWindow();
|
page = await app.firstWindow();
|
||||||
|
// main.js opens DevTools in dev mode, and that window can be the first one registered.
|
||||||
|
while (page.url().startsWith('devtools://')) {
|
||||||
|
page = app.windows().find((w) => !w.url().startsWith('devtools://'))
|
||||||
|
?? await new Promise((resolve) => app.once('window', resolve));
|
||||||
|
}
|
||||||
page.on('console', (msg) => logs.push(`[renderer] ${msg.type()}: ${msg.text()}`));
|
page.on('console', (msg) => logs.push(`[renderer] ${msg.type()}: ${msg.text()}`));
|
||||||
|
|
||||||
await page.waitForSelector('input');
|
|
||||||
await chooseServer(HOMESERVER);
|
await chooseServer(HOMESERVER);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Config-based screenshots and traces do not fire for a page taken from app.firstWindow(),
|
||||||
|
// and console capture is not covered by config at all, so this hook stays manual.
|
||||||
test.afterEach(async ({}, testInfo) => {
|
test.afterEach(async ({}, testInfo) => {
|
||||||
|
if (!app) return;
|
||||||
|
|
||||||
if (testInfo.status !== testInfo.expectedStatus) {
|
if (testInfo.status !== testInfo.expectedStatus) {
|
||||||
if (page && !page.isClosed()) {
|
if (page && !page.isClosed()) {
|
||||||
await testInfo.attach('failure.png', { body: await page.screenshot(), contentType: 'image/png' });
|
await testInfo.attach('failure.png', { body: await page.screenshot(), contentType: 'image/png' });
|
||||||
@@ -61,20 +80,30 @@ test.afterEach(async ({}, testInfo) => {
|
|||||||
.close()
|
.close()
|
||||||
// main.js preventDefault()s window close to minimise to tray, so close can hang.
|
// main.js preventDefault()s window close to minimise to tray, so close can hang.
|
||||||
.catch(() => app.process().kill('SIGTERM'));
|
.catch(() => app.process().kill('SIGTERM'));
|
||||||
|
|
||||||
|
fs.rmSync(profile, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
async function expectAuthenticated() {
|
async function expectAuthenticated() {
|
||||||
// Login complete reloads into the client shell, which lands on the last active
|
// Login complete reloads into the client shell, which lands on the last active path
|
||||||
// path or the inbox, so assert on the shell rather than one route.
|
// (state/lastActivePath.ts) or the inbox, so assert a shell route rather than any URL.
|
||||||
await expect(page).not.toHaveURL(/#\/login\//, { timeout: 30_000 });
|
await expect(page).toHaveURL(/#\/(inbox|home|room|space)/);
|
||||||
|
|
||||||
const session = await page.evaluate(() => ({
|
await expect
|
||||||
userId: JSON.parse(localStorage.getItem('currentSessionUserId') ?? 'null'),
|
.poll(() => page.evaluate(() => JSON.parse(localStorage.getItem('currentSessionUserId') ?? 'null')))
|
||||||
sessions: JSON.parse(localStorage.getItem('matrixSessions') ?? '[]'),
|
.toBe(`@${USER}:${HOMESERVER}`);
|
||||||
}));
|
|
||||||
|
|
||||||
expect(session.userId).toBe(`@${USER}:${HOMESERVER}`);
|
// A truthy accessToken only proves a string exists, so prove it authenticates. Only the
|
||||||
expect(session.sessions[0].accessToken).toBeTruthy();
|
// status is asserted, so a failure cannot put a live token into the CI log or the report.
|
||||||
|
await expect
|
||||||
|
.poll(async () => {
|
||||||
|
const token = await page.evaluate(() => JSON.parse(localStorage.getItem('matrixSessions') ?? '[]')[0]?.accessToken);
|
||||||
|
if (!token) return 0;
|
||||||
|
// /versions and /account 404 on ruv.wtf; /capabilities exists and requires a valid token.
|
||||||
|
const res = await fetch(`https://${HOMESERVER}/_matrix/client/v3/capabilities`, { headers: { Authorization: `Bearer ${token}` } });
|
||||||
|
return res.status;
|
||||||
|
})
|
||||||
|
.toBe(200);
|
||||||
|
|
||||||
// The login form is gone, so the app is on the authenticated shell.
|
// The login form is gone, so the app is on the authenticated shell.
|
||||||
await expect(usernameInput()).toHaveCount(0);
|
await expect(usernameInput()).toHaveCount(0);
|
||||||
@@ -88,12 +117,10 @@ test('test account can log in and reaches the authenticated app', async () => {
|
|||||||
test('failed login reports an error without crashing the app', async () => {
|
test('failed login reports an error without crashing the app', async () => {
|
||||||
await submitLogin(USER, 'not-the-password');
|
await submitLogin(USER, 'not-the-password');
|
||||||
|
|
||||||
await expect(page.getByText('Invalid Username or Password.')).toBeVisible({ timeout: 30_000 });
|
await expect(page.getByText('Invalid Username or Password.')).toBeVisible();
|
||||||
|
|
||||||
// The main process is still running and the form is still usable.
|
// The renderer is still alive and the form is still usable. No second login here: every
|
||||||
expect(app.process().exitCode).toBeNull();
|
// run costs Synapse rc_login budget, and the re-login was the main source of flakes.
|
||||||
|
await expect(page.evaluate(() => document.body.innerText.trim())).not.toBe('');
|
||||||
await expect(usernameInput()).toBeVisible();
|
await expect(usernameInput()).toBeVisible();
|
||||||
|
|
||||||
await submitLogin(USER, PASSWORD);
|
|
||||||
await expectAuthenticated();
|
|
||||||
});
|
});
|
||||||
|
|||||||
+1
-1
@@ -2,7 +2,7 @@ import { defineConfig } from 'vitest/config';
|
|||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
test: {
|
test: {
|
||||||
|
// The default pattern would also pick up the Electron spec in tests/e2e.
|
||||||
include: ['tests/unit/**/*.test.js'],
|
include: ['tests/unit/**/*.test.js'],
|
||||||
environment: 'node',
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user