From b5eb76fc957fa7d76b55cb1f8716eb69cc8c8b13 Mon Sep 17 00:00:00 2001 From: Brian Vaughn Date: Sun, 23 Jun 2024 13:09:29 -0700 Subject: [PATCH 1/3] Update README to mention Node v18 requirement and PNPM --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 553fc759..af825804 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,15 @@ Docs are built with: ## Installation ```bash -npm i +pnpm install ``` ## Local development +Note that Node 18 is currently required to run this web server successfully. + ``` -npm run dev +pnpm run dev ``` ## Issues From 0623f9b0b9d3a64e5e04525985e1b814a1549b6b Mon Sep 17 00:00:00 2001 From: Brian Vaughn Date: Fri, 5 Jul 2024 10:35:29 -0400 Subject: [PATCH 2/3] Update CI "Upload strategies" page to mention new advanced upload options --- .../ci-workflows/upload-strategies/page.md | 264 +++++++++++------- 1 file changed, 156 insertions(+), 108 deletions(-) diff --git a/src/app/reference/ci-workflows/upload-strategies/page.md b/src/app/reference/ci-workflows/upload-strategies/page.md index ee7ad244..3946e4a0 100644 --- a/src/app/reference/ci-workflows/upload-strategies/page.md +++ b/src/app/reference/ci-workflows/upload-strategies/page.md @@ -1,34 +1,27 @@ --- title: Upload strategies -description: When you run tests and create recordings, they are stored locally. You can opt to upload them automatically or define your own uploading strategy. All uploaded recordings become accessible in the Replay App. +description: When you run tests and create recordings, they are stored locally. You can choose which recordings get uploaded to Replay. All uploaded recordings become accessible in the Replay App. --- {% callout %} -While uploading just failed test is good for saving resources, our recommendation is to upload both failed and passed tests so that you can compare them. This can be really useful for debugging purposes. +While uploading just failed tests is good for saving resources, our recommendation is to upload both failed and passed tests so that you can compare them. This can be really useful for debugging purposes. {% /callout %} -## Upload failed tests only +## Upload all tests -By default, all test replays are uploaded no matter the result. If you want to upload only the failed recordings, you can use the `filter` property in the plugin configuration: +To upload all test replays no matter the result, set the `upload` option to true. {% tabs labels=["cypress", "playwright"] %} {% tab %} -```js +```js {% fileName="cypress.config.js" highlight=[7] %} export default defineConfig({ e2e: { setupNodeEvents(cyOn, config) { const on = wrapOn(cyOn) replayPlugin(on, config, { - upload: true, apiKey: process.env.REPLAY_API_KEY, - filter: function (recording) { - // upload runtime crashes and any failed tests - return ( - recording.status === 'crashed' || - recording.metadata.test.result === 'failed' - ) - }, + upload: true, }) return config }, @@ -39,7 +32,7 @@ export default defineConfig({ {% /tab %} {% tab %} -```js +```js {% fileName="playwright.config.js" highlight=[7] %} import { replayDevices, replayReporter } from "@replayio/playwright"; const config: PlaywrightTestConfig = { @@ -47,13 +40,6 @@ const config: PlaywrightTestConfig = { replayReporter({ apiKey: process.env.REPLAY_API_KEY, upload: true, - filter: function (recording) { - // upload runtime crashes and any failed tests - return ( - recording.status === "crashed" || - recording.metadata.test.result === "failed" - ); - }, }), ["line"], ], @@ -64,32 +50,29 @@ const config: PlaywrightTestConfig = { }, ], }; + export default config; ``` {% /tab %} {% /tabs %} -## Upload failed and flaky Cypress tests +## Upload failed tests only + +To upload recordings only for failed tests use `statusThreshold` option: -By default, all test replays are uploaded no matter the result. If you want to upload failed and flaky tests, you can use the `filter` property in the plugin configuration: +{% tabs labels=["cypress", "playwright"] %} +{% tab %} -```js +```js {% fileName="cypress.config.js" highlight=[7,8,9] %} export default defineConfig({ e2e: { setupNodeEvents(cyOn, config) { const on = wrapOn(cyOn) replayPlugin(on, config, { - upload: true, apiKey: process.env.REPLAY_API_KEY, - filter: function (recording) { - // upload runtime crashes and recordings with any tests that failed - return ( - recording.status === 'crashed' || - recording.metadata.test.tests.some( - (test) => test.result === 'failed', - ) - ) + upload: { + statusThreshold: 'failed', }, }) return config @@ -98,23 +81,52 @@ export default defineConfig({ }) ``` -## Upload only for the primary branch +{% /tab %} +{% tab %} -The recording metadata includes some details about the source control including the repository and branch name which can also be used to filter your uploads. The example below uploads all recordings from the `main` branch: +```js {% fileName="playwright.config.js" highlight=[7,8,9] %} +import { replayDevices, replayReporter } from "@replayio/playwright"; + +const config: PlaywrightTestConfig = { + reporter: [ + replayReporter({ + apiKey: process.env.REPLAY_API_KEY, + upload: { + statusThreshold: "failed" + }, + }), + ["line"], + ], + projects: [ + { + name: "replay-chromium", + use: { ...replayDevices["Replay Chromium"] }, + }, + ], +}; + +export default config; +``` + +{% /tab %} +{% /tabs %} + +## Upload failed and flaky tests + +To upload recordings for failed and flaky tests use `statusThreshold` option: {% tabs labels=["cypress", "playwright"] %} {% tab %} -```js +```js {% fileName="cypress.config.js" highlight=[7,8,9] %} export default defineConfig({ e2e: { setupNodeEvents(cyOn, config) { const on = wrapOn(cyOn) replayPlugin(on, config, { - upload: true, apiKey: process.env.REPLAY_API_KEY, - filter: function (recording) { - return recording.metadata.source.branch === 'main' + upload: { + statusThreshold: 'failed-and-flaky', }, }) return config @@ -126,17 +138,16 @@ export default defineConfig({ {% /tab %} {% tab %} -```js +```js {% fileName="playwright.config.js" highlight=[7,8,9] %} import { replayDevices, replayReporter } from "@replayio/playwright"; const config: PlaywrightTestConfig = { reporter: [ replayReporter({ apiKey: process.env.REPLAY_API_KEY, - upload: true, - filter: function (recording) { - return recording.metadata.source.branch === "main"; - }, + upload: { + statusThreshold: "failed-and-flaky" + }, }), ["line"], ], @@ -147,42 +158,35 @@ const config: PlaywrightTestConfig = { }, ], }; + export default config; ``` {% /tab %} {% /tabs %} -## Upload some passing runs +## Upload only for the primary branch -If you've adopted one the configurations above but would also like to periodically upload all replays for a test run, you can add a condition to the filter that returns `true` for a given test run id. This is only one possible implementation of this approach and you're welcome to adopt others such as using external environment variables. +Many CI providers provide an environment variable that references the current branch name. + +- [CircleCI](https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables): `$CIRCLE_BRANCH` +- [GitLab](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html): `$CI_COMMIT_REF_NAME` +- [Semaphore](https://docs.semaphoreci.com/ci-cd-environment/environment-variables/): `$SEMAPHORE_GIT_BRANCH` +- [Travis](https://docs.travis-ci.com/user/environment-variables/#default-environment-variables): `$TRAVIS_BRANCH` + +GitHub [stores this value in a default variable](https://docs.github.com/en/actions/learn-github-actions/variables) named `GITHUB_BASE_REF` that can be passed along as [part of a Workflow](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow) and then referenced in the test config like so: {% tabs labels=["cypress", "playwright"] %} {% tab %} -```js -const convertStringToInt = (string) => - string.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) - +```js {% fileName="cypress.config.js" highlight=[7] %} export default defineConfig({ e2e: { setupNodeEvents(cyOn, config) { const on = wrapOn(cyOn) replayPlugin(on, config, { - upload: true, apiKey: process.env.REPLAY_API_KEY, - filter: function (recording) { - // randomly upload 10% of all test runs - if (convertStringToInt(r.metadata.test.run.id) % 10 === 1) { - return true - } - - // upload runtime crashes and any failed tests - return ( - recording.status === 'crashed' || - recording.metadata.test.result === 'failed' - ) - }, + upload: process.env.BRANCH_NAME === 'main', }) return config }, @@ -193,29 +197,14 @@ export default defineConfig({ {% /tab %} {% tab %} -```js +```js {% fileName="playwright.config.js" highlight=[7] %} import { replayDevices, replayReporter } from "@replayio/playwright"; -const convertStringToInt = string => - string.split("").reduce((acc, char) => acc + char.charCodeAt(0), 0); - const config: PlaywrightTestConfig = { reporter: [ replayReporter({ apiKey: process.env.REPLAY_API_KEY, - upload: true, - filter: function (recording) { - // randomly upload 10% of all test runs - if (convertStringToInt(r.metadata.test.run.id) % 10 === 1) { - return true; - } - - // upload runtime crashes and any failed tests - return ( - recording.status === "crashed" || - recording.metadata.test.result === "failed" - ); - }, + upload: process.env.BRANCH_NAME === "main" }), ["line"], ], @@ -226,38 +215,97 @@ const config: PlaywrightTestConfig = { }, ], }; + export default config; ``` {% /tab %} {% /tabs %} -## Using GitHub Action - -Alternatively, you can upload your replays in a separate step using our [GitHub upload action](https://github.com/replayio/action-upload). To filter which replays to upload, you can use [JSONata filtering functions](https://docs.jsonata.org/higher-order-functions#filter). - -```yml {% fileName=".github/workflows/e2e.yml" lineNumbers=true highlight=[19] %} -name: Replay tests -on: - pull_request: - push: - branches: [main] -jobs: - cypress-run: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install dependencies - run: npm ci - - name: Install Replay Chromium - run: npx replayio install - - name: Run Playwright tests with Replay Browser - run: npx playwright test --project replay-chromium --reporter=@replayio/playwright/reporter,line - - name: Upload replays - if: ${{ always() }} - uses: replayio/action-upload@v0.5.1 - with: - api-key: ${{ secrets.REPLAY_API_KEY }} - filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }} +## Reducing the number of uploaded recordings + +Use the advanced upload options to reduce the number of recordings that are uploaded. When this option is enabled, only one recording will be uploaded for any passing or failing test. For flaky tests, two recordings will be uploaded– the passing test and one of the failed attempts. + +```js {% fileName="playwright.config.js" highlight=[7,8,9] %} +import { replayDevices, replayReporter } from "@replayio/playwright"; + +const config: PlaywrightTestConfig = { + reporter: [ + replayReporter({ + apiKey: process.env.REPLAY_API_KEY, + upload: { + minimizeUploads: true, + }, + }), + ["line"], + ], + projects: [ + { + name: "replay-chromium", + use: { ...replayDevices["Replay Chromium"] }, + }, + ], +}; + +export default config; +``` + +{% callout %} +Note this option is only available for tests recorded with Playwright +{% /callout %} + +## Combining options + +You can combine advanced upload options to e.g. only upload a single recording and only for a failing test. + +{% tabs labels=["cypress", "playwright"] %} +{% tab %} + +```js {% fileName="cypress.config.js" highlight=[7,8,9,10] %} +export default defineConfig({ + e2e: { + setupNodeEvents(cyOn, config) { + const on = wrapOn(cyOn) + replayPlugin(on, config, { + apiKey: process.env.REPLAY_API_KEY, + upload: { + minimizeUploads: true, + statusThreshold: 'failed', + }, + }) + return config + }, + }, +}) +``` + +{% /tab %} +{% tab %} + +```js {% fileName="playwright.config.js" highlight=[7,8,9,10] %} +import { replayDevices, replayReporter } from "@replayio/playwright"; + +const config: PlaywrightTestConfig = { + reporter: [ + replayReporter({ + apiKey: process.env.REPLAY_API_KEY, + upload: { + minimizeUploads: true, + statusThreshold: "failed", + }, + }), + ["line"], + ], + projects: [ + { + name: "replay-chromium", + use: { ...replayDevices["Replay Chromium"] }, + }, + ], +}; + +export default config; ``` + +{% /tab %} +{% /tabs %} From 840d2d29731ccad081dfe370e316ff6c4ac0ec12 Mon Sep 17 00:00:00 2001 From: Brian Vaughn Date: Fri, 5 Jul 2024 11:14:48 -0400 Subject: [PATCH 3/3] PR feedback --- .../ci-workflows/upload-strategies/page.md | 68 ++----------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/src/app/reference/ci-workflows/upload-strategies/page.md b/src/app/reference/ci-workflows/upload-strategies/page.md index 3946e4a0..1ab58b3d 100644 --- a/src/app/reference/ci-workflows/upload-strategies/page.md +++ b/src/app/reference/ci-workflows/upload-strategies/page.md @@ -92,8 +92,8 @@ const config: PlaywrightTestConfig = { replayReporter({ apiKey: process.env.REPLAY_API_KEY, upload: { - statusThreshold: "failed" - }, + statusThreshold: "failed" + }, }), ["line"], ], @@ -146,8 +146,8 @@ const config: PlaywrightTestConfig = { replayReporter({ apiKey: process.env.REPLAY_API_KEY, upload: { - statusThreshold: "failed-and-flaky" - }, + statusThreshold: "failed-and-flaky" + }, }), ["line"], ], @@ -234,8 +234,8 @@ const config: PlaywrightTestConfig = { replayReporter({ apiKey: process.env.REPLAY_API_KEY, upload: { - minimizeUploads: true, - }, + minimizeUploads: true, + }, }), ["line"], ], @@ -253,59 +253,3 @@ export default config; {% callout %} Note this option is only available for tests recorded with Playwright {% /callout %} - -## Combining options - -You can combine advanced upload options to e.g. only upload a single recording and only for a failing test. - -{% tabs labels=["cypress", "playwright"] %} -{% tab %} - -```js {% fileName="cypress.config.js" highlight=[7,8,9,10] %} -export default defineConfig({ - e2e: { - setupNodeEvents(cyOn, config) { - const on = wrapOn(cyOn) - replayPlugin(on, config, { - apiKey: process.env.REPLAY_API_KEY, - upload: { - minimizeUploads: true, - statusThreshold: 'failed', - }, - }) - return config - }, - }, -}) -``` - -{% /tab %} -{% tab %} - -```js {% fileName="playwright.config.js" highlight=[7,8,9,10] %} -import { replayDevices, replayReporter } from "@replayio/playwright"; - -const config: PlaywrightTestConfig = { - reporter: [ - replayReporter({ - apiKey: process.env.REPLAY_API_KEY, - upload: { - minimizeUploads: true, - statusThreshold: "failed", - }, - }), - ["line"], - ], - projects: [ - { - name: "replay-chromium", - use: { ...replayDevices["Replay Chromium"] }, - }, - ], -}; - -export default config; -``` - -{% /tab %} -{% /tabs %}