diff --git a/playwright.config.ts b/playwright.config.ts index d8a0e3ecf0..3906232abc 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -103,4 +103,5 @@ export default { outputDir: 'tests/e2e/test-artifacts/', /* Folder for explicit snapshots for visual testing */ snapshotDir: 'tests/e2e/test-snapshots/', + snapshotPathTemplate: '{snapshotDir}/snapshots/{testFilePath}/{projectName}_{arg}{ext}', } satisfies PlaywrightTestConfig; diff --git a/tests/e2e/README.md b/tests/e2e/README.md index cc255e05d9..8d8858bfd5 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -155,20 +155,6 @@ For SQLite: make test-e2e-sqlite#example ``` -### Visual testing - -> **Warning** -> This is not currently used by most Forgejo contributors. -> Your help to improve the situation and allow for visual testing is appreciated. - -Although the main goal of e2e is assertion testing, we have added a framework for visual regression testing. If you are working on front-end features, please use the following: - - Check out `main`, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1` to generate outputs. This will initially fail, as no screenshots exist. You can run the e2e tests again to assert that it passes. - - Check out your branch, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1`. You should be able to assert that your front-end changes don't break any other tests unintentionally. - -`VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. The test will fail the first time this is enabled (until we get visual test image persistence figured out), because it will be testing against an empty screenshot folder. - -`ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. - ## Tips and tricks @@ -216,6 +202,41 @@ you can alternatively use: await page.waitForURL('**/target.html'); ~~~ +### Visual testing + +Due to size and frequent updates, we do not host screenshots in the Forgejo repository. +However, it is good practice to ensure that your test is capable of generating relevant and stable screenshots. +Forgejo is regularly tested against visual regressions in a dedicated repository which contains the screenshots: +https://code.forgejo.org/forgejo/visual-browser-testing/ + +For tests that consume only the `page`, +screenshots are automatically created at the end of each test. + +If your test visits different relevant screens or pages during the test, +or creates a custom `page` from context +(e.g. for tests that require a signed-in user) +calling `await save_visual(page);` explicitly in relevant positions is encouraged. + +Please confirm locally that your screenshots are stable by performing several runs of your test. +When screenshots are available and reproducible, +check in your test without the screenshots. + +When your screenshots differ between runs, +for example because dynamic elements (e.g. timestamps, commit hashes etc) +change between runs, +mask these elements in the `save_visual` function in `utils_e2e.ts`. + +#### Working with screenshots + +The following environment variables control visual testing: + +`VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. + The test will fail the first time, + because the screenshots are not included with Forgejo. + Subsequent runs will comopare against your local copy of the screenshots. + +`ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. + ### Only sign in if necessary Signing in takes time and is actually executed step-by-step. diff --git a/tests/e2e/actions.test.e2e.ts b/tests/e2e/actions.test.e2e.ts index e9f7db52f2..a66b608080 100644 --- a/tests/e2e/actions.test.e2e.ts +++ b/tests/e2e/actions.test.e2e.ts @@ -10,7 +10,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -33,6 +33,7 @@ test('workflow dispatch present', async ({browser}, workerInfo) => { await expect(menu).toBeHidden(); await run_workflow_btn.click(); await expect(menu).toBeVisible(); + await save_visual(page); }); test('workflow dispatch error: missing inputs', async ({browser}, workerInfo) => { @@ -54,6 +55,7 @@ test('workflow dispatch error: missing inputs', async ({browser}, workerInfo) => await page.locator('#workflow-dispatch-submit').click(); await expect(page.getByText('Require value for input "String w/o. default".')).toBeVisible(); + await save_visual(page); }); test('workflow dispatch success', async ({browser}, workerInfo) => { @@ -67,11 +69,13 @@ test('workflow dispatch success', async ({browser}, workerInfo) => { await page.locator('#workflow_dispatch_dropdown>button').click(); await page.fill('input[name="inputs[string2]"]', 'abc'); + await save_visual(page); await page.locator('#workflow-dispatch-submit').click(); await expect(page.getByText('Workflow run was successfully requested.')).toBeVisible(); await expect(page.locator('.run-list>:first-child .run-list-meta', {hasText: 'now'})).toBeVisible(); + await save_visual(page); }); test('workflow dispatch box not available for unauthenticated users', async ({page}) => { diff --git a/tests/e2e/dashboard-ci-status.test.e2e.ts b/tests/e2e/dashboard-ci-status.test.e2e.ts index e25403ad9a..1d23122b44 100644 --- a/tests/e2e/dashboard-ci-status.test.e2e.ts +++ b/tests/e2e/dashboard-ci-status.test.e2e.ts @@ -3,7 +3,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -20,4 +20,5 @@ test('Correct link and tooltip', async ({browser}, workerInfo) => { await page.waitForLoadState('networkidle'); // eslint-disable-line playwright/no-networkidle await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000}); await expect(repoStatus).toHaveAttribute('data-tooltip-content', /^(Error|Failure)$/); + await save_visual(page); }); diff --git a/tests/e2e/example.test.e2e.ts b/tests/e2e/example.test.e2e.ts index 64818c4557..b2a679a82d 100644 --- a/tests/e2e/example.test.e2e.ts +++ b/tests/e2e/example.test.e2e.ts @@ -5,11 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, save_visual} from './utils_e2e.ts'; - -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +import {test} from './utils_e2e.ts'; test('Load Homepage', async ({page}) => { const response = await page.goto('/'); @@ -30,8 +26,6 @@ test('Register Form', async ({page}, workerInfo) => { expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`); await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible(); await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!'); - - save_visual(page); }); // eslint-disable-next-line playwright/no-skipped-test diff --git a/tests/e2e/git-notes.test.e2e.ts b/tests/e2e/git-notes.test.e2e.ts index c0dc1618db..8b80a3aa77 100644 --- a/tests/e2e/git-notes.test.e2e.ts +++ b/tests/e2e/git-notes.test.e2e.ts @@ -1,6 +1,6 @@ // @ts-check import {test, expect} from '@playwright/test'; -import {login_user, load_logged_in_context} from './utils_e2e.ts'; +import {login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -17,14 +17,15 @@ test('Change git note', async ({browser}, workerInfo) => { let textarea = page.locator('textarea[name="notes"]'); await expect(textarea).toBeVisible(); await textarea.fill('This is a new note'); + await save_visual(page); await page.locator('#notes-save-button').click(); - - expect(response?.status()).toBe(200); + await save_visual(page); response = await page.goto('/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d'); expect(response?.status()).toBe(200); textarea = page.locator('textarea[name="notes"]'); await expect(textarea).toHaveText('This is a new note'); + await save_visual(page); }); diff --git a/tests/e2e/issue-comment.test.e2e.ts b/tests/e2e/issue-comment.test.e2e.ts index 9a3a45f522..f493f64506 100644 --- a/tests/e2e/issue-comment.test.e2e.ts +++ b/tests/e2e/issue-comment.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, login} from './utils_e2e.ts'; +import {test, save_visual, login_user, login} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -66,6 +66,7 @@ test('Always focus edit tab first on edit', async ({browser}, workerInfo) => { await expect(editTab).toHaveClass(/active/); await expect(previewTab).not.toHaveClass(/active/); + await save_visual(page); }); test('Quote reply', async ({browser}, workerInfo) => { diff --git a/tests/e2e/issue-sidebar.test.e2e.ts b/tests/e2e/issue-sidebar.test.e2e.ts index 1e05069e7f..f4d50a13ba 100644 --- a/tests/e2e/issue-sidebar.test.e2e.ts +++ b/tests/e2e/issue-sidebar.test.e2e.ts @@ -7,7 +7,7 @@ /* eslint playwright/expect-expect: ["error", { "assertFunctionNames": ["check_wip"] }] */ import {expect, type Page} from '@playwright/test'; -import {test, login_user, login} from './utils_e2e.ts'; +import {test, save_visual, login_user, login} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -203,6 +203,7 @@ test('New Issue: Assignees', async ({browser}, workerInfo) => { await page.locator('.select-assignees .menu .item').filter({hasText: 'user4'}).click(); await page.locator('.select-assignees.dropdown').click(); await expect(assigneesList.filter({hasText: 'user4'})).toBeVisible(); + await save_visual(page); // remove user4 await page.locator('.select-assignees.dropdown').click(); @@ -220,6 +221,7 @@ test('New Issue: Assignees', async ({browser}, workerInfo) => { await page.fill('.select-assignees .menu .search input', ''); await page.locator('.select-assignees.dropdown .no-select.item').click(); await expect(page.locator('.select-assign-me')).toBeVisible(); + await save_visual(page); }); test('Issue: Milestone', async ({browser}, workerInfo) => { @@ -256,14 +258,17 @@ test('New Issue: Milestone', async ({browser}, workerInfo) => { const selectedMilestone = page.locator('.issue-content-right .select-milestone.list'); const milestoneDropdown = page.locator('.issue-content-right .select-milestone.dropdown'); await expect(selectedMilestone).toContainText('No milestone'); + await save_visual(page); // Add milestone. await milestoneDropdown.click(); await page.getByRole('option', {name: 'milestone1'}).click(); await expect(selectedMilestone).toContainText('milestone1'); + await save_visual(page); // Clear milestone. await milestoneDropdown.click(); await page.getByText('Clear milestone', {exact: true}).click(); await expect(selectedMilestone).toContainText('No milestone'); + await save_visual(page); }); diff --git a/tests/e2e/markdown-editor.test.e2e.ts b/tests/e2e/markdown-editor.test.e2e.ts index 582ea7eceb..ca2d6e01b6 100644 --- a/tests/e2e/markdown-editor.test.e2e.ts +++ b/tests/e2e/markdown-editor.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, load_logged_in_context, login_user} from './utils_e2e.ts'; +import {test, save_visual, load_logged_in_context, login_user} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -40,6 +40,7 @@ test('Markdown image preview behaviour', async ({browser}, workerInfo) => { // Check for the image preview via the expected attribute const preview = page.locator('div[data-tab="preview"] p[dir="auto"] a'); await expect(preview).toHaveAttribute('href', 'http://localhost:3003/user2/repo1/media/branch/master/assets/logo.svg'); + await save_visual(page); }); test('markdown indentation', async ({browser}, workerInfo) => { @@ -224,6 +225,7 @@ test('markdown insert table', async ({browser}, workerInfo) => { const newTableModal = page.locator('div[data-markdown-table-modal-id="0"]'); await expect(newTableModal).toBeVisible(); + await save_visual(page); await newTableModal.locator('input[name="table-rows"]').fill('3'); await newTableModal.locator('input[name="table-columns"]').fill('2'); @@ -234,4 +236,5 @@ test('markdown insert table', async ({browser}, workerInfo) => { const textarea = page.locator('textarea[name=content]'); await expect(textarea).toHaveValue('| Header | Header |\n|---------|---------|\n| Content | Content |\n| Content | Content |\n| Content | Content |\n'); + await save_visual(page); }); diff --git a/tests/e2e/org-settings.test.e2e.ts b/tests/e2e/org-settings.test.e2e.ts index b645d94161..22a8bc0e2d 100644 --- a/tests/e2e/org-settings.test.e2e.ts +++ b/tests/e2e/org-settings.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, login} from './utils_e2e.ts'; +import {test, save_visual, login_user, login} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; test.beforeAll(async ({browser}, workerInfo) => { @@ -20,9 +20,11 @@ test('org team settings', async ({browser}, workerInfo) => { await page.locator('input[name="permission"][value="admin"]').click(); await expect(page.locator('.hide-unless-checked')).toBeHidden(); + await save_visual(page); await page.locator('input[name="permission"][value="read"]').click(); await expect(page.locator('.hide-unless-checked')).toBeVisible(); + await save_visual(page); // we are validating the form here to include the part that could be hidden await validate_form({page}); diff --git a/tests/e2e/profile_actions.test.e2e.ts b/tests/e2e/profile_actions.test.e2e.ts index 51a690aa60..65090e62b2 100644 --- a/tests/e2e/profile_actions.test.e2e.ts +++ b/tests/e2e/profile_actions.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; test('Follow actions', async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -29,6 +29,7 @@ test('Follow actions', async ({browser}, workerInfo) => { await page.locator('.block').click(); await expect(page.locator('#block-user')).toBeVisible(); + await save_visual(page); await page.locator('#block-user .ok').click(); await expect(page.locator('.block')).toContainText('Unblock'); await expect(page.locator('#block-user')).toBeHidden(); @@ -38,6 +39,7 @@ test('Follow actions', async ({browser}, workerInfo) => { const flashMessage = page.locator('#flash-message'); await expect(flashMessage).toBeVisible(); await expect(flashMessage).toContainText('You cannot follow this user because you have blocked this user or this user has blocked you.'); + await save_visual(page); // Unblock interaction. await page.locator('.block').click(); diff --git a/tests/e2e/reaction-selectors.test.e2e.ts b/tests/e2e/reaction-selectors.test.e2e.ts index 3bd54c7881..3ce71b24d7 100644 --- a/tests/e2e/reaction-selectors.test.e2e.ts +++ b/tests/e2e/reaction-selectors.test.e2e.ts @@ -4,7 +4,7 @@ // @watch end import {expect, type Locator} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; test.beforeAll(async ({browser}, workerInfo) => { await login_user(browser, workerInfo, 'user2'); @@ -66,4 +66,5 @@ test('Reaction Selectors', async ({browser}, workerInfo) => { await toggleReaction(topPicker, 'laugh'); await assertReactionCounts(comment, {'laugh': 2}); + await save_visual(page); }); diff --git a/tests/e2e/release.test.e2e.ts b/tests/e2e/release.test.e2e.ts index 373f23dfa7..fefa446c59 100644 --- a/tests/e2e/release.test.e2e.ts +++ b/tests/e2e/release.test.e2e.ts @@ -41,7 +41,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await page.fill('input[name=attachment-new-name-2]', 'Test'); await page.fill('input[name=attachment-new-exturl-2]', 'https://forgejo.org/'); await page.click('.remove-rel-attach'); - save_visual(page); + await save_visual(page); await page.click('.button.small.primary'); // Validate release page and click edit @@ -53,7 +53,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.download[open] li:nth-of-type(2) a')).toHaveAttribute('href', '/user2/repo2/archive/2.0.tar.gz'); await expect(page.locator('.download[open] li:nth-of-type(3)')).toContainText('Test'); await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://forgejo.org/'); - save_visual(page); + await save_visual(page); await page.locator('.octicon-pencil').first().click(); // Validate edit page and edit the release @@ -68,7 +68,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.attachment_edit:visible')).toHaveCount(4); await page.locator('.attachment_edit:visible').nth(2).fill('Test3'); await page.locator('.attachment_edit:visible').nth(3).fill('https://gitea.com/'); - save_visual(page); + await save_visual(page); await page.click('.button.small.primary'); // Validate release page and click edit @@ -78,7 +78,7 @@ test('External Release Attachments', async ({browser, isMobile}, workerInfo) => await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://gitea.io/'); await expect(page.locator('.download[open] li:nth-of-type(4)')).toContainText('Test3'); await expect(page.locator('.download[open] li:nth-of-type(4) a')).toHaveAttribute('href', 'https://gitea.com/'); - save_visual(page); + await save_visual(page); await page.locator('.octicon-pencil').first().click(); // Delete release diff --git a/tests/e2e/repo-code.test.e2e.ts b/tests/e2e/repo-code.test.e2e.ts index d78fa33fe5..264dd3a8e0 100644 --- a/tests/e2e/repo-code.test.e2e.ts +++ b/tests/e2e/repo-code.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect, type Page} from '@playwright/test'; -import {test, login_user, login} from './utils_e2e.ts'; +import {test, save_visual, login_user, login} from './utils_e2e.ts'; import {accessibilityCheck} from './shared/accessibility.ts'; test.beforeAll(async ({browser}, workerInfo) => { @@ -89,10 +89,12 @@ test('Username highlighted in commits', async ({browser}, workerInfo) => { await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); await accessibilityCheck({page}, ['.commit-header'], [], []); + await save_visual(page); // check second commit await page.goto('/user2/mentions-highlighted/commits/branch/main'); await page.locator('tbody').getByRole('link', {name: 'Another commit which mentions'}).click(); await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); await accessibilityCheck({page}, ['.commit-header'], [], []); + await save_visual(page); }); diff --git a/tests/e2e/repo-migrate.test.e2e.ts b/tests/e2e/repo-migrate.test.e2e.ts index c4d1604a6f..a0f9ab6c80 100644 --- a/tests/e2e/repo-migrate.test.e2e.ts +++ b/tests/e2e/repo-migrate.test.e2e.ts @@ -3,7 +3,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; test.beforeAll(({browser}, workerInfo) => login_user(browser, workerInfo, 'user2')); @@ -19,17 +19,22 @@ test('Migration Progress Page', async ({page: unauthedPage, browser}, workerInfo const form = page.locator('form'); await form.getByRole('textbox', {name: 'Repository Name'}).fill('invalidrepo'); await form.getByRole('textbox', {name: 'Migrate / Clone from URL'}).fill('https://codeberg.org/forgejo/invalidrepo'); + await save_visual(page); await form.locator('button.primary').click({timeout: 5000}); await expect(page).toHaveURL('user2/invalidrepo'); + await save_visual(page); + // page screenshot of unauthedPage is checked automatically after the test expect((await unauthedPage.goto('/user2/invalidrepo'))?.status(), 'public migration page should be accessible').toBe(200); await expect(unauthedPage.locator('#repo_migrating_progress')).toBeVisible(); await page.reload(); await expect(page.locator('#repo_migrating_failed')).toBeVisible(); + await save_visual(page); await page.getByRole('button', {name: 'Delete this repository'}).click(); const deleteModal = page.locator('#delete-repo-modal'); await deleteModal.getByRole('textbox', {name: 'Confirmation string'}).fill('user2/invalidrepo'); + await save_visual(page); await deleteModal.getByRole('button', {name: 'Delete repository'}).click(); await expect(page).toHaveURL('/'); }); diff --git a/tests/e2e/repo-settings.test.e2e.ts b/tests/e2e/repo-settings.test.e2e.ts index 8bd7299182..113b15181b 100644 --- a/tests/e2e/repo-settings.test.e2e.ts +++ b/tests/e2e/repo-settings.test.e2e.ts @@ -7,7 +7,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, login} from './utils_e2e.ts'; +import {test, save_visual, login_user, login} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; test.beforeAll(async ({browser}, workerInfo) => { @@ -25,11 +25,13 @@ test('repo webhook settings', async ({browser}, workerInfo) => { // check accessibility including the custom events (now visible) part await validate_form({page}, 'fieldset'); + await save_visual(page); await page.locator('input[name="events"][value="push_only"]').click(); await expect(page.locator('.hide-unless-checked')).toBeHidden(); await page.locator('input[name="events"][value="send_everything"]').click(); await expect(page.locator('.hide-unless-checked')).toBeHidden(); + await save_visual(page); }); test.describe('repo branch protection settings', () => { @@ -44,11 +46,14 @@ test.describe('repo branch protection settings', () => { // verify header is new await expect(page.locator('h4')).toContainText('new'); await page.locator('input[name="rule_name"]').fill('testrule'); + await save_visual(page); await page.getByText('Save rule').click(); // verify header is in edit mode await page.waitForLoadState('domcontentloaded'); + await save_visual(page); await page.getByText('Edit').click(); await expect(page.locator('h4')).toContainText('Protection rules for branch'); + await save_visual(page); }); test.afterEach(async ({browser}, workerInfo) => { diff --git a/tests/e2e/utils_e2e.ts b/tests/e2e/utils_e2e.ts index a52495bcc6..09189e6826 100644 --- a/tests/e2e/utils_e2e.ts +++ b/tests/e2e/utils_e2e.ts @@ -4,6 +4,15 @@ export const test = baseTest.extend({ context: async ({browser}, use) => { return use(await test_context(browser)); }, + // see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks + forEachTest: [async ({page}, use) => { + await use(); + // some tests create a new page which is not yet available here + // only operate on tests that make the URL available + if (page.url() !== 'about:blank') { + await save_visual(page); + } + }, {auto: true}], }); async function test_context(browser: Browser, options?: BrowserContextOptions) { @@ -66,14 +75,28 @@ export async function save_visual(page: Page) { // Optionally include visual testing if (process.env.VISUAL_TEST) { await page.waitForLoadState('domcontentloaded'); - // Mock page/version string - await page.locator('footer div.ui.left').evaluate((node) => node.innerHTML = 'MOCK'); + // Mock/replace dynamic content which can have different size (and thus cannot simply be masked below) + await page.locator('footer .left-links').evaluate((node) => node.innerHTML = 'MOCK'); + // replace timestamps in repos to mask them later down + await page.locator('.flex-item-body > relative-time').filter({hasText: /now|minute/}).evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'relative time in repo'; + }); + await page.locator('relative-time').evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'time element'; + }); + // used for instance for security keys + await page.locator('absolute-date').evaluateAll((nodes) => { + for (const node of nodes) node.outerHTML = 'time element'; + }); await expect(page).toHaveScreenshot({ fullPage: true, timeout: 20000, mask: [ - page.locator('.secondary-nav span>img.ui.avatar'), - page.locator('.ui.dropdown.jump.item span>img.ui.avatar'), + page.locator('.ui.avatar'), + page.locator('.sha'), + page.locator('#repo_migrating'), + // update order of recently created repos is not fully deterministic + page.locator('.flex-item-main').filter({hasText: 'relative time in repo'}), ], }); } diff --git a/tests/e2e/webauthn.test.e2e.ts b/tests/e2e/webauthn.test.e2e.ts index 98a2d1c152..144f52e374 100644 --- a/tests/e2e/webauthn.test.e2e.ts +++ b/tests/e2e/webauthn.test.e2e.ts @@ -8,7 +8,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test, create_temp_user, login_user} from './utils_e2e.ts'; +import {test, save_visual, create_temp_user, login_user} from './utils_e2e.ts'; test('WebAuthn register & login flow', async ({browser, request}, workerInfo) => { test.skip(workerInfo.project.name !== 'chromium', 'Uses Chrome protocol'); @@ -34,6 +34,7 @@ test('WebAuthn register & login flow', async ({browser, request}, workerInfo) => }); await page.locator('input#nickname').fill('Testing Security Key'); + await save_visual(page); await page.getByText('Add security key').click(); // Logout. @@ -57,6 +58,7 @@ test('WebAuthn register & login flow', async ({browser, request}, workerInfo) => response = await page.goto('/user/settings/security'); expect(response?.status()).toBe(200); await page.getByRole('button', {name: 'Remove'}).click(); + await save_visual(page); await page.getByRole('button', {name: 'Yes'}).click(); await page.waitForLoadState();