diff --git a/playwright/docker-compose.yml b/playwright/docker-compose.yml index e5dd4a70..72736eb5 100644 --- a/playwright/docker-compose.yml +++ b/playwright/docker-compose.yml @@ -24,6 +24,7 @@ services: environment: - DATABASE_URL - I_REALLY_WANT_VOLATILE_STORAGE + - LOGIN_RATELIMIT_MAX_BURST - SMTP_HOST - SMTP_FROM - SMTP_DEBUG diff --git a/playwright/global-utils.ts b/playwright/global-utils.ts index 79f4f90b..317a69b4 100644 --- a/playwright/global-utils.ts +++ b/playwright/global-utils.ts @@ -189,7 +189,7 @@ export async function startVaultwarden(browser: Browser, testInfo: TestInfo, env console.log(`Starting Vaultwarden`); execSync(`docker compose --profile playwright --env-file test.env up -d Vaultwarden`, { - env: { ...env, ...dbConfig(testInfo) }, + env: { LOGIN_RATELIMIT_MAX_BURST: 100, ...env, ...dbConfig(testInfo) }, }); await waitFor("/", browser); console.log(`Vaultwarden running on: ${process.env.DOMAIN}`); diff --git a/playwright/package-lock.json b/playwright/package-lock.json index ab05d130..2a278ee9 100644 --- a/playwright/package-lock.json +++ b/playwright/package-lock.json @@ -9,15 +9,15 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "mysql2": "^3.14.0", + "mysql2": "^3.14.1", "otpauth": "^9.4.0", - "pg": "^8.14.1" + "pg": "^8.15.6" }, "devDependencies": { - "@playwright/test": "^1.51.1", + "@playwright/test": "^1.52.0", "dotenv": "^16.5.0", "dotenv-expand": "^12.0.2", - "maildev": "github:timshel/maildev#3.0.4" + "maildev": "github:timshel/maildev#3.0.5" } }, "node_modules/@asamuzakjp/css-color": { @@ -155,12 +155,12 @@ } }, "node_modules/@playwright/test": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz", - "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz", + "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==", "dev": true, "dependencies": { - "playwright": "1.51.1" + "playwright": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -1484,9 +1484,9 @@ "dev": true }, "node_modules/mysql2": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.0.tgz", - "integrity": "sha512-8eMhmG6gt/hRkU1G+8KlGOdQi2w+CgtNoD1ksXZq9gQfkfDsX4LHaBwTe1SY0Imx//t2iZA03DFnyYKPinxSRw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.1.tgz", + "integrity": "sha512-7ytuPQJjQB8TNAYX/H2yhL+iQOnIBjAMam361R7UAL0lOVXWjtdrmoL9HYKqKoLp/8UUTRcvo1QPvK9KL7wA8w==", "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", @@ -1648,13 +1648,13 @@ } }, "node_modules/pg": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.1.tgz", - "integrity": "sha512-0TdbqfjwIun9Fm/r89oB7RFQ0bLgduAhiIqIXOsyKoiC/L54DbuAAzIEN/9Op0f1Po9X7iCPXGoa/Ah+2aI8Xw==", + "version": "8.15.6", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.15.6.tgz", + "integrity": "sha512-yvao7YI3GdmmrslNVsZgx9PfntfWrnXwtR+K/DjI0I/sTKif4Z623um+sjVZ1hk5670B+ODjvHDAckKdjmPTsg==", "dependencies": { - "pg-connection-string": "^2.7.0", - "pg-pool": "^3.8.0", - "pg-protocol": "^1.8.0", + "pg-connection-string": "^2.8.5", + "pg-pool": "^3.9.6", + "pg-protocol": "^1.9.5", "pg-types": "^2.1.0", "pgpass": "1.x" }, @@ -1662,7 +1662,7 @@ "node": ">= 8.0.0" }, "optionalDependencies": { - "pg-cloudflare": "^1.1.1" + "pg-cloudflare": "^1.2.5" }, "peerDependencies": { "pg-native": ">=3.0.1" @@ -1674,15 +1674,15 @@ } }, "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz", + "integrity": "sha512-OOX22Vt0vOSRrdoUPKJ8Wi2OpE/o/h9T8X1s4qSkCedbNah9ei2W2765be8iMVxQUsvgT7zIAT2eIa9fs5+vtg==", "optional": true }, "node_modules/pg-connection-string": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz", - "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==" + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.8.5.tgz", + "integrity": "sha512-Ni8FuZ8yAF+sWZzojvtLE2b03cqjO5jNULcHFfM9ZZ0/JXrgom5pBREbtnAw7oxsxJqHw9Nz/XWORUEL3/IFow==" }, "node_modules/pg-int8": { "version": "1.0.1", @@ -1693,17 +1693,17 @@ } }, "node_modules/pg-pool": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz", - "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==", + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.9.6.tgz", + "integrity": "sha512-rFen0G7adh1YmgvrmE5IPIqbb+IgEzENUm+tzm6MLLDSlPRoZVhzU1WdML9PV2W5GOdRA9qBKURlbt1OsXOsPw==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz", - "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==" + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.9.5.tgz", + "integrity": "sha512-DYTWtWpfd5FOro3UnAfwvhD8jh59r2ig8bPtc9H8Ds7MscE/9NYruUQWFAOuraRl29jwcT2kyMFQ3MxeaVjUhg==" }, "node_modules/pg-types": { "version": "2.2.0", @@ -1729,12 +1729,12 @@ } }, "node_modules/playwright": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", - "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz", + "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==", "dev": true, "dependencies": { - "playwright-core": "1.51.1" + "playwright-core": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -1747,9 +1747,9 @@ } }, "node_modules/playwright-core": { - "version": "1.51.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz", - "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz", + "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==", "dev": true, "bin": { "playwright-core": "cli.js" diff --git a/playwright/package.json b/playwright/package.json index e014accc..4dea6b86 100644 --- a/playwright/package.json +++ b/playwright/package.json @@ -8,14 +8,14 @@ "author": "", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.51.1", + "@playwright/test": "^1.52.0", "dotenv": "^16.5.0", "dotenv-expand": "^12.0.2", - "maildev": "github:timshel/maildev#3.0.4" + "maildev": "github:timshel/maildev#3.0.5" }, "dependencies": { - "mysql2": "^3.14.0", + "mysql2": "^3.14.1", "otpauth": "^9.4.0", - "pg": "^8.14.1" + "pg": "^8.15.6" } } diff --git a/playwright/playwright.config.ts b/playwright/playwright.config.ts index ff7f5001..0f0df9c2 100644 --- a/playwright/playwright.config.ts +++ b/playwright/playwright.config.ts @@ -16,7 +16,6 @@ export default defineConfig({ /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, - /* Retry on CI only */ retries: 0, workers: 1, @@ -38,11 +37,12 @@ export default defineConfig({ browserName: 'firefox', locale: 'en-GB', timezoneId: 'Europe/London', - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + + /* Always collect trace (other values add random test failures) See https://playwright.dev/docs/trace-viewer */ + trace: 'on', viewport: { - width: 1920, - height: 1080 + width: 1080, + height: 720, }, video: "on", }, diff --git a/playwright/tests/organization.smtp.spec.ts b/playwright/tests/organization.smtp.spec.ts index 3f4c999b..547213d4 100644 --- a/playwright/tests/organization.smtp.spec.ts +++ b/playwright/tests/organization.smtp.spec.ts @@ -103,7 +103,7 @@ test('Confirm invited user', async ({ page }) => { await logUser(test, page, users.user1, mail1Buffer); await orgs.members(test, page, 'Test'); - await orgs.confirm(test, page, 'Test', users.user2.name); + await orgs.confirm(test, page, 'Test', users.user2.email); await expect(mail2Buffer.next((m) => m.subject.includes('Invitation to Test confirmed'))).resolves.toBeDefined(); }); diff --git a/playwright/tests/organization.spec.ts b/playwright/tests/organization.spec.ts index 2750257e..5c4dcd39 100644 --- a/playwright/tests/organization.spec.ts +++ b/playwright/tests/organization.spec.ts @@ -2,6 +2,7 @@ import { test, expect, type TestInfo } from '@playwright/test'; import { MailDev } from 'maildev'; import * as utils from "../global-utils"; +import * as orgs from './setups/orgs'; import { createAccount, logUser } from './setups/user'; let users = utils.loadEnv(); @@ -14,77 +15,40 @@ test.afterAll('Teardown', async ({}, testInfo: TestInfo) => { utils.stopVaultwarden(testInfo); }); -test('Create user3', async ({ page }) => { +test('Invite', async ({ page }) => { await createAccount(test, page, users.user3); -}); - -test('Invite users', async ({ page }) => { await createAccount(test, page, users.user1); - await test.step('Create Org', async () => { - await page.getByRole('link', { name: 'New organisation' }).click(); - await page.getByLabel('Organisation name (required)').fill('Test'); - await page.getByRole('button', { name: 'Submit' }).click(); - await page.locator('div').filter({ hasText: 'Members' }).nth(2).click(); - }); + await orgs.create(test, page, 'New organisation'); + await orgs.members(test, page, 'New organisation'); - await test.step('Invite user2', async () => { - await page.getByRole('button', { name: 'Invite member' }).click(); - await page.getByLabel('Email (required)').fill(users.user2.email); - await page.getByRole('tab', { name: 'Collections' }).click(); - await page.getByLabel('Permission').selectOption('edit'); - await page.getByLabel('Select collections').click(); - await page.getByLabel('Options list').getByText('Default collection').click(); - await page.getByRole('button', { name: 'Save' }).click(); - await utils.checkNotification(page, 'User(s) invited'); + await test.step('missing user2', async () => { + await orgs.invite(test, page, 'New organisation', users.user2.email); await expect(page.getByRole('row', { name: users.user2.email })).toHaveText(/Invited/); }); - await test.step('Invite user3', async () => { - await page.getByRole('button', { name: 'Invite member' }).click(); - await page.getByLabel('Email (required)').fill(users.user3.email); - await page.getByRole('tab', { name: 'Collections' }).click(); - await page.getByLabel('Permission').selectOption('edit'); - await page.getByLabel('Select collections').click(); - await page.getByLabel('Options list').getByText('Default collection').click(); - await page.getByRole('button', { name: 'Save' }).click(); - await utils.checkNotification(page, 'User(s) invited'); - await expect(page.getByRole('row', { name: users.user3.name })).toHaveText(/Needs confirmation/); + await test.step('existing user3', async () => { + await orgs.invite(test, page, 'New organisation', users.user3.email); + await expect(page.getByRole('row', { name: users.user3.email })).toHaveText(/Needs confirmation/); + await orgs.confirm(test, page, 'New organisation', users.user3.email); }); - await test.step('Confirm existing user3', async () => { - await page.getByRole('row', { name: users.user3.name }).getByLabel('Options').click(); - await page.getByRole('menuitem', { name: 'Confirm' }).click(); - await page.getByRole('button', { name: 'Confirm' }).click(); - await utils.checkNotification(page, 'confirmed'); + await test.step('confirm user2', async () => { + await createAccount(test, page, users.user2); + await logUser(test, page, users.user1); + await orgs.members(test, page, 'New organisation'); + await orgs.confirm(test, page, 'New organisation', users.user2.email); + }); + + await test.step('Org visible user2 ', async () => { + await logUser(test, page, users.user2); + await page.getByRole('button', { name: 'vault: New organisation', exact: true }).click(); + await expect(page.getByLabel('Filter: Default collection')).toBeVisible(); + }); + + await test.step('Org visible user3 ', async () => { + await logUser(test, page, users.user3); + await page.getByRole('button', { name: 'vault: New organisation', exact: true }).click(); + await expect(page.getByLabel('Filter: Default collection')).toBeVisible(); }); }); - -test('Create invited account', async ({ page }) => { - await createAccount(test, page, users.user2); -}); - -test('Confirm invited user', async ({ page }) => { - await logUser(test, page, users.user1); - await page.getByLabel('Switch products').click(); - await page.getByRole('link', { name: ' Admin Console' }).click(); - await page.getByRole('link', { name: 'Members' }).click(); - - await test.step('Confirm user2', async () => { - await page.getByRole('row', { name: users.user2.name }).getByLabel('Options').click(); - await page.getByRole('menuitem', { name: 'Confirm' }).click(); - await page.getByRole('button', { name: 'Confirm' }).click(); - await utils.checkNotification(page, 'confirmed'); - }); -}); - -test('Organization is visible', async ({ context, page }) => { - await logUser(test, page, users.user2); - await page.getByLabel('vault: Test').click(); - await expect(page.getByLabel('Filter: Default collection')).toBeVisible(); - - const page2 = await context.newPage(); - await logUser(test, page2, users.user3); - await page2.getByLabel('vault: Test').click(); - await expect(page2.getByLabel('Filter: Default collection')).toBeVisible(); -}); diff --git a/playwright/tests/setups/orgs.ts b/playwright/tests/setups/orgs.ts index a0f5299b..e38cb278 100644 --- a/playwright/tests/setups/orgs.ts +++ b/playwright/tests/setups/orgs.ts @@ -22,6 +22,7 @@ export async function members(test, page: Page, name: string) { await expect(page.getByRole('heading', { name: `${name} collections` })).toBeVisible(); await page.locator('div').filter({ hasText: 'Members' }).nth(2).click(); await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible(); + await expect(page.getByRole('cell', { name: 'All' })).toBeVisible(); }); } @@ -39,10 +40,10 @@ export async function invite(test, page: Page, name: string, email: string) { }); } -export async function confirm(test, page: Page, name: string, user_name: string) { - await test.step(`Confirm ${user_name}`, async () => { +export async function confirm(test, page: Page, name: string, user_email: string) { + await test.step(`Confirm ${user_email}`, async () => { await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible(); - await page.getByRole('row', { name: user_name }).getByLabel('Options').click(); + await page.getByRole('row').filter({hasText: user_email}).getByLabel('Options').click(); await page.getByRole('menuitem', { name: 'Confirm' }).click(); await expect(page.getByRole('heading', { name: 'Confirm user' })).toBeVisible(); await page.getByRole('button', { name: 'Confirm' }).click(); @@ -50,10 +51,10 @@ export async function confirm(test, page: Page, name: string, user_name: string) }); } -export async function revoke(test, page: Page, name: string, user_name: string) { - await test.step(`Revoke ${user_name}`, async () => { +export async function revoke(test, page: Page, name: string, user_email: string) { + await test.step(`Revoke ${user_email}`, async () => { await expect(page.getByRole('heading', { name: 'Members' })).toBeVisible(); - await page.getByRole('row', { name: user_name }).getByLabel('Options').click(); + await page.getByRole('row').filter({hasText: user_email}).getByLabel('Options').click(); await page.getByRole('menuitem', { name: 'Revoke access' }).click(); await expect(page.getByRole('heading', { name: 'Revoke access' })).toBeVisible(); await page.getByRole('button', { name: 'Revoke access' }).click(); diff --git a/playwright/tests/setups/sso.ts b/playwright/tests/setups/sso.ts index 52438bd2..faf9e742 100644 --- a/playwright/tests/setups/sso.ts +++ b/playwright/tests/setups/sso.ts @@ -15,6 +15,8 @@ export async function logNewUser( let mailBuffer = options.mailBuffer ?? options.mailServer?.buffer(user.email); try { await test.step('Create user', async () => { + await page.context().clearCookies(); + await test.step('Landing page', async () => { await page.goto('/'); await page.getByLabel(/Email address/).fill(user.email); @@ -66,6 +68,8 @@ export async function logUser( let mailBuffer = options.mailBuffer ?? options.mailServer?.buffer(user.email); try { await test.step('Log user', async () => { + await page.context().clearCookies(); + await test.step('Landing page', async () => { await page.goto('/'); await page.getByLabel(/Email address/).fill(user.email); diff --git a/playwright/tests/setups/user.ts b/playwright/tests/setups/user.ts index b9f79eb9..6c18dc92 100644 --- a/playwright/tests/setups/user.ts +++ b/playwright/tests/setups/user.ts @@ -6,7 +6,15 @@ import * as utils from '../../global-utils'; export async function createAccount(test, page: Page, user: { email: string, name: string, password: string }, mailBuffer?: MailBuffer) { await test.step('Create user', async () => { // Landing page - await page.goto('/'); + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await expect(page.getByRole('button').nth(0)).toBeVisible(); + + const logged = await page.getByRole('button', { name: 'Log out' }).count(); + if( logged > 0 ){ + await page.getByRole('button', { name: 'Log out' }).click(); + await page.getByRole('button', { name: 'Log out' }).click(); + } + await page.getByRole('link', { name: 'Create account' }).click(); // Back to Vault create account @@ -33,7 +41,15 @@ export async function createAccount(test, page: Page, user: { email: string, nam export async function logUser(test, page: Page, user: { email: string, password: string }, mailBuffer?: MailBuffer) { await test.step('Log user', async () => { // Landing page - await page.goto('/'); + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await expect(page.getByRole('button').nth(0)).toBeVisible(); + + const logged = await page.getByRole('button', { name: 'Log out' }).count(); + if( logged > 0 ){ + await page.getByRole('button', { name: 'Log out' }).click(); + await page.getByRole('button', { name: 'Log out' }).click(); + } + await page.getByLabel(/Email address/).fill(user.email); await page.getByRole('button', { name: 'Continue' }).click();