diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index e13c135..32e7f67 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -22,6 +22,9 @@ jobs: run: npm run test:unit e2e: + # A fork PR controls both the test files and these steps, so only same-repository + # PRs can be trusted with the E2E_PASSWORD secret. + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest env: # Dedicated test account on the homeserver, not a personal one. @@ -30,33 +33,16 @@ jobs: # Repo secret: E2E_PASSWORD. The test fails with a clear message when it is missing. E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} steps: + # .gitmodules uses a relative URL, so the runner resolves it against origin. - name: Checkout repository uses: actions/checkout@v4 with: - submodules: false - - - name: Checkout submodules manually - run: | - set -euo pipefail - # Use the same host/scheme the runner already used for origin (not public HTTPS). - ORIGIN_URL=$(git remote get-url origin) - ORIGIN_URL=${ORIGIN_URL%/} - ORIGIN_URL=${ORIGIN_URL%.git} - BASE_URL=${ORIGIN_URL%/*} - CINNY_URL="${BASE_URL}/cinny.git" - echo "Rewriting submodule URL to ${CINNY_URL}" - git config submodule.cinny.url "${CINNY_URL}" - git submodule sync --recursive - git submodule update --init --recursive + submodules: true - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '22' - cache: 'npm' - cache-dependency-path: | - package-lock.json - cinny/package-lock.json - name: Install dependencies run: npm ci diff --git a/README.md b/README.md index 2c20cfa..d484883 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ The E2E test drives the real Electron app against the vite dev server, so it nee cinny dependencies installed and an X display. On a headless machine run it under xvfb: `xvfb-run -a npm run test:e2e`. -The login test uses a dedicated test account, never a personal one: +The E2E test logs in to a live homeserver over the network, so it needs network access and a +working test account. It uses a dedicated account, never a personal one: Variable | Default ---|--- diff --git a/playwright.config.mjs b/playwright.config.mjs index 5657ce1..2300866 100644 --- a/playwright.config.mjs +++ b/playwright.config.mjs @@ -4,10 +4,19 @@ export default defineConfig({ testDir: './tests/e2e', // Electron launch plus first paint is far slower than a page load. timeout: 120_000, + expect: { timeout: 30_000 }, // One Electron process at a time: the app takes a single-instance lock and // every test shares the same vite dev server. workers: 1, - outputDir: 'test-results', - reporter: [['list'], ['html', { open: false }]], - globalSetup: './tests/e2e/dev-server.mjs', + // The suite logs in to a live homeserver, where Synapse login rate limiting is a real flake. + retries: process.env.CI ? 1 : 0, + reporter: [['list'], ['html']], + // cinny/vite.config.js binds 0.0.0.0 with fs.allow: ['..'], which is exposed on a shared + // CI runner, so pin the dev server to loopback. The flag has to reach vite itself: + // `npm run dev:vite -- --host` lands on sh's positional params, not on vite. + // localhost is also the URL electron/main.js loads, so the page keeps that origin. + webServer: { + command: 'sh -c "cd cinny && npm start -- --host localhost"', + url: 'http://localhost:38347', + }, }); diff --git a/tests/e2e/dev-server.mjs b/tests/e2e/dev-server.mjs deleted file mode 100644 index 210a757..0000000 --- a/tests/e2e/dev-server.mjs +++ /dev/null @@ -1,36 +0,0 @@ -import { spawn } from 'node:child_process'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -// electron/main.js loads this URL whenever the app is not packaged, so the E2E -// run needs the vite dev server up. It is the same server `npm run dev` uses. -export const DEV_URL = 'http://localhost:38347'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); - -const reachable = async () => { - try { - const res = await fetch(DEV_URL, { signal: AbortSignal.timeout(3000) }); - return res.ok; - } catch { - return false; - } -}; - -export default async function globalSetup() { - // A dev server the developer already started is reused rather than restarted. - if (await reachable()) return () => {}; - - const child = spawn('npm', ['run', 'dev:vite'], { cwd: ROOT, detached: true, stdio: 'inherit' }); - - for (let attempt = 0; attempt < 60; attempt++) { - if (await reachable()) { - // detached puts npm and its sh -c child in their own process group. - return () => process.kill(-child.pid, 'SIGTERM'); - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - process.kill(-child.pid, 'SIGTERM'); - throw new Error(`Vite dev server did not come up at ${DEV_URL}`); -} diff --git a/tests/e2e/login.spec.mjs b/tests/e2e/login.spec.mjs index dcf9ed0..0a1a296 100644 --- a/tests/e2e/login.spec.mjs +++ b/tests/e2e/login.spec.mjs @@ -9,47 +9,66 @@ const PASSWORD = process.env.E2E_PASSWORD; let app; let page; -let logs; +let logs = []; +let profile; +// ServerPicker gives its input no name or test id, so position is the only handle on it; +// this breaks if the form ever gains a field before the ones below. const serverInput = () => page.locator('input').first(); const usernameInput = () => page.locator('input[name="usernameInput"]'); const passwordInput = () => page.locator('input[name="passwordInput"]'); +const submitButton = () => page.locator('button[type="submit"]'); + +async function expectLoginForm() { + await expect(serverInput()).toBeVisible(); + await expect(usernameInput()).toBeVisible(); + await expect(passwordInput()).toBeVisible(); + await expect(submitButton()).toBeVisible(); +} async function chooseServer(server) { await serverInput().fill(server); // The server field commits on Enter, which re-runs discovery and reloads the auth flows. await serverInput().press('Enter'); - await expect(usernameInput()).toBeVisible(); + await expectLoginForm(); } async function submitLogin(username, password) { await usernameInput().fill(username); await passwordInput().fill(password); - await page.locator('button[type="submit"]').click(); + await submitButton().click(); } test.beforeEach(async () => { + logs = []; + if (!PASSWORD) { throw new Error('E2E_PASSWORD is not set. Export it, or add it as the CI secret.'); } - logs = []; - // Fresh profile per run. electron/main.js takes a single-instance lock keyed to // the user data dir, so a shared profile collides with a running Paarrot. - const profile = fs.mkdtempSync(path.join(os.tmpdir(), 'paarrot-e2e-')); + profile = fs.mkdtempSync(path.join(os.tmpdir(), 'paarrot-e2e-')); app = await _electron.launch({ args: ['.', `--user-data-dir=${profile}`] }); app.on('console', (msg) => logs.push(`[main] ${msg.type()}: ${msg.text()}`)); page = await app.firstWindow(); + // main.js opens DevTools in dev mode, and that window can be the first one registered. + while (page.url().startsWith('devtools://')) { + page = app.windows().find((w) => !w.url().startsWith('devtools://')) + ?? await new Promise((resolve) => app.once('window', resolve)); + } page.on('console', (msg) => logs.push(`[renderer] ${msg.type()}: ${msg.text()}`)); - await page.waitForSelector('input'); await chooseServer(HOMESERVER); }); +// Config-based screenshots and traces do not fire for a page taken from app.firstWindow(), +// and console capture is not covered by config at all, so this hook stays manual. test.afterEach(async ({}, testInfo) => { + if (!app) return; + if (testInfo.status !== testInfo.expectedStatus) { if (page && !page.isClosed()) { await testInfo.attach('failure.png', { body: await page.screenshot(), contentType: 'image/png' }); @@ -61,20 +80,30 @@ test.afterEach(async ({}, testInfo) => { .close() // main.js preventDefault()s window close to minimise to tray, so close can hang. .catch(() => app.process().kill('SIGTERM')); + + fs.rmSync(profile, { recursive: true, force: true }); }); async function expectAuthenticated() { - // Login complete reloads into the client shell, which lands on the last active - // path or the inbox, so assert on the shell rather than one route. - await expect(page).not.toHaveURL(/#\/login\//, { timeout: 30_000 }); + // Login complete reloads into the client shell, which lands on the last active path + // (state/lastActivePath.ts) or the inbox, so assert a shell route rather than any URL. + await expect(page).toHaveURL(/#\/(inbox|home|room|space)/); - const session = await page.evaluate(() => ({ - userId: JSON.parse(localStorage.getItem('currentSessionUserId') ?? 'null'), - sessions: JSON.parse(localStorage.getItem('matrixSessions') ?? '[]'), - })); + await expect + .poll(() => page.evaluate(() => JSON.parse(localStorage.getItem('currentSessionUserId') ?? 'null'))) + .toBe(`@${USER}:${HOMESERVER}`); - expect(session.userId).toBe(`@${USER}:${HOMESERVER}`); - expect(session.sessions[0].accessToken).toBeTruthy(); + // A truthy accessToken only proves a string exists, so prove it authenticates. Only the + // status is asserted, so a failure cannot put a live token into the CI log or the report. + await expect + .poll(async () => { + const token = await page.evaluate(() => JSON.parse(localStorage.getItem('matrixSessions') ?? '[]')[0]?.accessToken); + if (!token) return 0; + // /versions and /account 404 on ruv.wtf; /capabilities exists and requires a valid token. + const res = await fetch(`https://${HOMESERVER}/_matrix/client/v3/capabilities`, { headers: { Authorization: `Bearer ${token}` } }); + return res.status; + }) + .toBe(200); // The login form is gone, so the app is on the authenticated shell. await expect(usernameInput()).toHaveCount(0); @@ -88,12 +117,10 @@ test('test account can log in and reaches the authenticated app', async () => { test('failed login reports an error without crashing the app', async () => { await submitLogin(USER, 'not-the-password'); - await expect(page.getByText('Invalid Username or Password.')).toBeVisible({ timeout: 30_000 }); + await expect(page.getByText('Invalid Username or Password.')).toBeVisible(); - // The main process is still running and the form is still usable. - expect(app.process().exitCode).toBeNull(); + // The renderer is still alive and the form is still usable. No second login here: every + // run costs Synapse rc_login budget, and the re-login was the main source of flakes. + await expect(page.evaluate(() => document.body.innerText.trim())).not.toBe(''); await expect(usernameInput()).toBeVisible(); - - await submitLogin(USER, PASSWORD); - await expectAuthenticated(); }); diff --git a/vitest.config.mjs b/vitest.config.mjs index fbdc945..7bb5cc8 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -2,7 +2,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // The default pattern would also pick up the Electron spec in tests/e2e. include: ['tests/unit/**/*.test.js'], - environment: 'node', }, });