Playwright tests improvements

2025-07-06 04:25:00 +00:00 · 2025-05-08 11:47:50 +02:00 · 2025-05-08 11:47:50 +02:00 · a6c8fb742d
commit a6c8fb742d
parent cbdf403ef0
10 changed files with 105 additions and 119 deletions
--- a/playwright/docker-compose.yml
+++ b/playwright/docker-compose.yml
@ -24,6 +24,7 @@ services:
    environment:
      - DATABASE_URL
      - I_REALLY_WANT_VOLATILE_STORAGE
+      - LOGIN_RATELIMIT_MAX_BURST
      - SMTP_HOST
      - SMTP_FROM
      - SMTP_DEBUG
--- a/playwright/global-utils.ts
+++ b/playwright/global-utils.ts
@ -189,7 +189,7 @@ export async function startVaultwarden(browser: Browser, testInfo: TestInfo, env

    console.log(`Starting Vaultwarden`);
    execSync(`docker compose --profile playwright --env-file test.env up -d Vaultwarden`, {
-        env: { ...env, ...dbConfig(testInfo) },
+        env: { LOGIN_RATELIMIT_MAX_BURST: 100, ...env, ...dbConfig(testInfo) },
    });
    await waitFor("/", browser);
    console.log(`Vaultwarden running on: ${process.env.DOMAIN}`);
--- a/playwright/package-lock.json
+++ b/playwright/package-lock.json
@ -9,15 +9,15 @@
            "version": "1.0.0",
            "license": "ISC",
            "dependencies": {
-                "mysql2": "^3.14.0",
+                "mysql2": "^3.14.1",
                "otpauth": "^9.4.0",
-                "pg": "^8.14.1"
+                "pg": "^8.15.6"
            },
            "devDependencies": {
-                "@playwright/test": "^1.51.1",
+                "@playwright/test": "^1.52.0",
                "dotenv": "^16.5.0",
                "dotenv-expand": "^12.0.2",
-                "maildev": "github:timshel/maildev#3.0.4"
+                "maildev": "github:timshel/maildev#3.0.5"
            }
        },
        "node_modules/@asamuzakjp/css-color": {
@ -155,12 +155,12 @@
            }
        },
        "node_modules/@playwright/test": {
-            "version": "1.51.1",
-            "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz",
-            "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==",
+            "version": "1.52.0",
+            "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz",
+            "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==",
            "dev": true,
            "dependencies": {
-                "playwright": "1.51.1"
+                "playwright": "1.52.0"
            },
            "bin": {
                "playwright": "cli.js"
@ -1484,9 +1484,9 @@
            "dev": true
        },
        "node_modules/mysql2": {
-            "version": "3.14.0",
-            "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.0.tgz",
-            "integrity": "sha512-8eMhmG6gt/hRkU1G+8KlGOdQi2w+CgtNoD1ksXZq9gQfkfDsX4LHaBwTe1SY0Imx//t2iZA03DFnyYKPinxSRw==",
+            "version": "3.14.1",
+            "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.1.tgz",
+            "integrity": "sha512-7ytuPQJjQB8TNAYX/H2yhL+iQOnIBjAMam361R7UAL0lOVXWjtdrmoL9HYKqKoLp/8UUTRcvo1QPvK9KL7wA8w==",
            "dependencies": {
                "aws-ssl-profiles": "^1.1.1",
                "denque": "^2.1.0",
@ -1648,13 +1648,13 @@
            }
        },
        "node_modules/pg": {
-            "version": "8.14.1",
-            "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.1.tgz",
-            "integrity": "sha512-0TdbqfjwIun9Fm/r89oB7RFQ0bLgduAhiIqIXOsyKoiC/L54DbuAAzIEN/9Op0f1Po9X7iCPXGoa/Ah+2aI8Xw==",
+            "version": "8.15.6",
+            "resolved": "https://registry.npmjs.org/pg/-/pg-8.15.6.tgz",
+            "integrity": "sha512-yvao7YI3GdmmrslNVsZgx9PfntfWrnXwtR+K/DjI0I/sTKif4Z623um+sjVZ1hk5670B+ODjvHDAckKdjmPTsg==",
            "dependencies": {
-                "pg-connection-string": "^2.7.0",
-                "pg-pool": "^3.8.0",
-                "pg-protocol": "^1.8.0",
+                "pg-connection-string": "^2.8.5",
+                "pg-pool": "^3.9.6",
+                "pg-protocol": "^1.9.5",
                "pg-types": "^2.1.0",
                "pgpass": "1.x"
            },
@ -1662,7 +1662,7 @@
                "node": ">= 8.0.0"
            },
            "optionalDependencies": {
-                "pg-cloudflare": "^1.1.1"
+                "pg-cloudflare": "^1.2.5"
            },
            "peerDependencies": {
                "pg-native": ">=3.0.1"
@ -1674,15 +1674,15 @@
            }
        },
        "node_modules/pg-cloudflare": {
-            "version": "1.1.1",
-            "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
-            "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+            "version": "1.2.5",
+            "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz",
+            "integrity": "sha512-OOX22Vt0vOSRrdoUPKJ8Wi2OpE/o/h9T8X1s4qSkCedbNah9ei2W2765be8iMVxQUsvgT7zIAT2eIa9fs5+vtg==",
            "optional": true
        },
        "node_modules/pg-connection-string": {
-            "version": "2.7.0",
-            "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz",
-            "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA=="
+            "version": "2.8.5",
+            "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.8.5.tgz",
+            "integrity": "sha512-Ni8FuZ8yAF+sWZzojvtLE2b03cqjO5jNULcHFfM9ZZ0/JXrgom5pBREbtnAw7oxsxJqHw9Nz/XWORUEL3/IFow=="
        },
        "node_modules/pg-int8": {
            "version": "1.0.1",
@ -1693,17 +1693,17 @@
            }
        },
        "node_modules/pg-pool": {
-            "version": "3.8.0",
-            "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz",
-            "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==",
+            "version": "3.9.6",
+            "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.9.6.tgz",
+            "integrity": "sha512-rFen0G7adh1YmgvrmE5IPIqbb+IgEzENUm+tzm6MLLDSlPRoZVhzU1WdML9PV2W5GOdRA9qBKURlbt1OsXOsPw==",
            "peerDependencies": {
                "pg": ">=8.0"
            }
        },
        "node_modules/pg-protocol": {
-            "version": "1.8.0",
-            "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz",
-            "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g=="
+            "version": "1.9.5",
+            "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.9.5.tgz",
+            "integrity": "sha512-DYTWtWpfd5FOro3UnAfwvhD8jh59r2ig8bPtc9H8Ds7MscE/9NYruUQWFAOuraRl29jwcT2kyMFQ3MxeaVjUhg=="
        },
        "node_modules/pg-types": {
            "version": "2.2.0",
@ -1729,12 +1729,12 @@
            }
        },
        "node_modules/playwright": {
-            "version": "1.51.1",
-            "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz",
-            "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==",
+            "version": "1.52.0",
+            "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz",
+            "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==",
            "dev": true,
            "dependencies": {
-                "playwright-core": "1.51.1"
+                "playwright-core": "1.52.0"
            },
            "bin": {
                "playwright": "cli.js"
@ -1747,9 +1747,9 @@
            }
        },
        "node_modules/playwright-core": {
-            "version": "1.51.1",
-            "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz",
-            "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==",
+            "version": "1.52.0",
+            "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz",
+            "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==",
            "dev": true,
            "bin": {
                "playwright-core": "cli.js"
--- a/playwright/package.json
+++ b/playwright/package.json
@ -8,14 +8,14 @@
    "author": "",
    "license": "ISC",
    "devDependencies": {
-        "@playwright/test": "^1.51.1",
+        "@playwright/test": "^1.52.0",
        "dotenv": "^16.5.0",
        "dotenv-expand": "^12.0.2",
-        "maildev": "github:timshel/maildev#3.0.4"
+        "maildev": "github:timshel/maildev#3.0.5"
    },
    "dependencies": {
-        "mysql2": "^3.14.0",
+        "mysql2": "^3.14.1",
        "otpauth": "^9.4.0",
-        "pg": "^8.14.1"
+        "pg": "^8.15.6"
    }
 }
--- a/playwright/playwright.config.ts
+++ b/playwright/playwright.config.ts
@ -16,7 +16,6 @@ export default defineConfig({
    /* Fail the build on CI if you accidentally left test.only in the source code. */
    forbidOnly: !!process.env.CI,

-    /* Retry on CI only */
    retries: 0,
    workers: 1,

@ -38,11 +37,12 @@ export default defineConfig({
        browserName: 'firefox',
        locale: 'en-GB',
        timezoneId: 'Europe/London',
-        /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-        trace: 'on-first-retry',
+
+        /* Always collect trace (other values add random test failures) See https://playwright.dev/docs/trace-viewer */
+        trace: 'on',
        viewport: {
-            width: 1920,
-            height: 1080
+            width: 1080,
+            height: 720,
        },
        video: "on",
    },
--- a/playwright/tests/organization.smtp.spec.ts
+++ b/playwright/tests/organization.smtp.spec.ts
@ -103,7 +103,7 @@ test('Confirm invited user', async ({ page }) => {
    await logUser(test, page, users.user1, mail1Buffer);

    await orgs.members(test, page, 'Test');
-    await orgs.confirm(test, page, 'Test', users.user2.name);
+    await orgs.confirm(test, page, 'Test', users.user2.email);

    await expect(mail2Buffer.next((m) => m.subject.includes('Invitation to Test confirmed'))).resolves.toBeDefined();
 });
--- a/playwright/tests/organization.spec.ts
+++ b/playwright/tests/organization.spec.ts
@ -2,6 +2,7 @@ import { test, expect, type TestInfo } from '@playwright/test';
 import { MailDev } from 'maildev';

 import * as utils from "../global-utils";
+import * as orgs from './setups/orgs';
 import { createAccount, logUser } from './setups/user';

 let users = utils.loadEnv();
@ -14,77 +15,40 @@ test.afterAll('Teardown', async ({}, testInfo: TestInfo) => {
    utils.stopVaultwarden(testInfo);
 });

-test('Create user3', async ({ page }) => {
+test('Invite', async ({ page }) => {
    await createAccount(test, page, users.user3);
-});
-
-test('Invite users', async ({ page }) => {
    await createAccount(test, page, users.user1);

-    await test.step('Create Org', async () => {
-        await page.getByRole('link', { name: 'New organisation' }).click();
-        await page.getByLabel('Organisation name (required)').fill('Test');
-        await page.getByRole('button', { name: 'Submit' }).click();
-        await page.locator('div').filter({ hasText: 'Members' }).nth(2).click();
-    });
+    await orgs.create(test, page, 'New organisation');
+    await orgs.members(test, page, 'New organisation');

-    await test.step('Invite user2', async () => {
-        await page.getByRole('button', { name: 'Invite member' }).click();
-        await page.getByLabel('Email (required)').fill(users.user2.email);
-        await page.getByRole('tab', { name: 'Collections' }).click();
-        await page.getByLabel('Permission').selectOption('edit');
-        await page.getByLabel('Select collections').click();
-        await page.getByLabel('Options list').getByText('Default collection').click();
-        await page.getByRole('button', { name: 'Save' }).click();
-        await utils.checkNotification(page, 'User(s) invited');
+    await test.step('missing user2', async () => {
+        await orgs.invite(test, page, 'New organisation', users.user2.email);
        await expect(page.getByRole('row', { name: users.user2.email })).toHaveText(/Invited/);
    });

-    await test.step('Invite user3', async () => {
-        await page.getByRole('button', { name: 'Invite member' }).click();
-        await page.getByLabel('Email (required)').fill(users.user3.email);
-        await page.getByRole('tab', { name: 'Collections' }).click();
-        await page.getByLabel('Permission').selectOption('edit');
-        await page.getByLabel('Select collections').click();
-        await page.getByLabel('Options list').getByText('Default collection').click();
-        await page.getByRole('button', { name: 'Save' }).click();
-        await utils.checkNotification(page, 'User(s) invited');
-        await expect(page.getByRole('row', { name: users.user3.name })).toHaveText(/Needs confirmation/);
+    await test.step('existing user3', async () => {
+        await orgs.invite(test, page, 'New organisation', users.user3.email);
+        await expect(page.getByRole('row', { name: users.user3.email })).toHaveText(/Needs confirmation/);
+        await orgs.confirm(test, page, 'New organisation', users.user3.email);
    });

-    await test.step('Confirm existing user3', async () => {
-        await page.getByRole('row', { name: users.user3.name }).getByLabel('Options').click();
-        await page.getByRole('menuitem', { name: 'Confirm' }).click();
-        await page.getByRole('button', { name: 'Confirm' }).click();
-        await utils.checkNotification(page, 'confirmed');
+    await test.step('confirm user2', async () => {
+        await createAccount(test, page, users.user2);
+        await logUser(test, page, users.user1);
+        await orgs.members(test, page, 'New organisation');
+        await orgs.confirm(test, page, 'New organisation', users.user2.email);
+    });
+
+    await test.step('Org visible user2  ', async () => {
+        await logUser(test, page, users.user2);
+        await page.getByRole('button', { name: 'vault: New organisation', exact: true }).click();
+        await expect(page.getByLabel('Filter: Default collection')).toBeVisible();
+    });
+
+    await test.step('Org visible user3  ', async () => {
+        await logUser(test, page, users.user3);
+        await page.getByRole('button', { name: 'vault: New organisation', exact: true }).click();
+        await expect(page.getByLabel('Filter: Default collection')).toBeVisible();
    });
 });
-
-test('Create invited account', async ({ page }) => {
-    await createAccount(test, page, users.user2);
-});
-
-test('Confirm invited user', async ({ page }) => {
-    await logUser(test, page, users.user1);
-    await page.getByLabel('Switch products').click();
-    await page.getByRole('link', { name: ' Admin Console' }).click();
-    await page.getByRole('link', { name: 'Members' }).click();
-
-    await test.step('Confirm user2', async () => {
-        await page.getByRole('row', { name: users.user2.name }).getByLabel('Options').click();
-        await page.getByRole('menuitem', { name: 'Confirm' }).click();
-        await page.getByRole('button', { name: 'Confirm' }).click();
-        await utils.checkNotification(page, 'confirmed');
-    });
-});
-
-test('Organization is visible', async ({ context, page }) => {
-    await logUser(test, page, users.user2);
-    await page.getByLabel('vault: Test').click();
-    await expect(page.getByLabel('Filter: Default collection')).toBeVisible();
-
-    const page2 = await context.newPage();
-    await logUser(test, page2, users.user3);
-    await page2.getByLabel('vault: Test').click();
-    await expect(page2.getByLabel('Filter: Default collection')).toBeVisible();
-});
--- a/playwright/tests/setups/orgs.ts
+++ b/playwright/tests/setups/orgs.ts
@ -22,6 +22,7 @@ export async function members(test, page: Page, name: string) {
        await expect(page.getByRole('heading', { name: `${name} collections` })).toBeVisible();
        await page.locator('div').filter({ hasText: 'Members' }).nth(2).click();
        await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible();
+        await expect(page.getByRole('cell', { name: 'All' })).toBeVisible();
    });
 }

@ -39,10 +40,10 @@ export async function invite(test, page: Page, name: string, email: string) {
    });
 }

-export async function confirm(test, page: Page, name: string, user_name: string) {
-    await test.step(`Confirm ${user_name}`, async () => {
+export async function confirm(test, page: Page, name: string, user_email: string) {
+    await test.step(`Confirm ${user_email}`, async () => {
        await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible();
-        await page.getByRole('row', { name: user_name }).getByLabel('Options').click();
+        await page.getByRole('row').filter({hasText: user_email}).getByLabel('Options').click();
        await page.getByRole('menuitem', { name: 'Confirm' }).click();
        await expect(page.getByRole('heading', { name: 'Confirm user' })).toBeVisible();
        await page.getByRole('button', { name: 'Confirm' }).click();
@ -50,10 +51,10 @@ export async function confirm(test, page: Page, name: string, user_name: string)
    });
 }

-export async function revoke(test, page: Page, name: string, user_name: string) {
-    await test.step(`Revoke ${user_name}`, async () => {
+export async function revoke(test, page: Page, name: string, user_email: string) {
+    await test.step(`Revoke ${user_email}`, async () => {
        await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible();
-        await page.getByRole('row', { name: user_name }).getByLabel('Options').click();
+        await page.getByRole('row').filter({hasText: user_email}).getByLabel('Options').click();
        await page.getByRole('menuitem', { name: 'Revoke access' }).click();
        await expect(page.getByRole('heading', { name: 'Revoke access' })).toBeVisible();
        await page.getByRole('button', { name: 'Revoke access' }).click();
--- a/playwright/tests/setups/sso.ts
+++ b/playwright/tests/setups/sso.ts
@ -15,6 +15,8 @@ export async function logNewUser(
    let mailBuffer = options.mailBuffer ?? options.mailServer?.buffer(user.email);
    try {
        await test.step('Create user', async () => {
+            await page.context().clearCookies();
+
            await test.step('Landing page', async () => {
                await page.goto('/');
                await page.getByLabel(/Email address/).fill(user.email);
@ -66,6 +68,8 @@ export async function logUser(
    let mailBuffer = options.mailBuffer ?? options.mailServer?.buffer(user.email);
    try {
        await test.step('Log user', async () => {
+            await page.context().clearCookies();
+
            await test.step('Landing page', async () => {
                await page.goto('/');
                await page.getByLabel(/Email address/).fill(user.email);
--- a/playwright/tests/setups/user.ts
+++ b/playwright/tests/setups/user.ts
@ -6,7 +6,15 @@ import * as utils from '../../global-utils';
 export async function createAccount(test, page: Page, user: { email: string, name: string, password: string }, mailBuffer?: MailBuffer) {
    await test.step('Create user', async () => {
        // Landing page
-        await page.goto('/');
+        await page.goto('/', { waitUntil: 'domcontentloaded' });
+        await expect(page.getByRole('button').nth(0)).toBeVisible();
+
+        const logged = await page.getByRole('button', { name: 'Log out' }).count();
+        if( logged > 0 ){
+            await page.getByRole('button', { name: 'Log out' }).click();
+            await page.getByRole('button', { name: 'Log out' }).click();
+        }
+
        await page.getByRole('link', { name: 'Create account' }).click();

        // Back to Vault create account
@ -33,7 +41,15 @@ export async function createAccount(test, page: Page, user: { email: string, nam
 export async function logUser(test, page: Page, user: { email: string, password: string }, mailBuffer?: MailBuffer) {
    await test.step('Log user', async () => {
        // Landing page
-        await page.goto('/');
+        await page.goto('/', { waitUntil: 'domcontentloaded' });
+        await expect(page.getByRole('button').nth(0)).toBeVisible();
+
+        const logged = await page.getByRole('button', { name: 'Log out' }).count();
+        if( logged > 0 ){
+            await page.getByRole('button', { name: 'Log out' }).click();
+            await page.getByRole('button', { name: 'Log out' }).click();
+        }
+
        await page.getByLabel(/Email address/).fill(user.email);
        await page.getByRole('button', { name: 'Continue' }).click();