diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 567f73c07a..44c670d275 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -20,4 +20,5 @@ Before submitting a PR, please read http://deno.land/manual/contributing
 6. Ensure `./tools/lint.js` passes.
 7. Open as a draft PR if your work is still in progress. The CI won't run
    all steps, but you can add '[ci]' to a commit message to force it to.
+8. If you would like to run the benchmarks on the CI, add the 'ci-bench' label.
 -->
diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts
index 69b07b7d20..49f1363c9e 100644
--- a/.github/workflows/ci.generate.ts
+++ b/.github/workflows/ci.generate.ts
@@ -192,7 +192,7 @@ const ci = {
   },
   concurrency: {
     group:
-      "${{ github.workflow }}-${{ !contains(github.event.pull_request.labels.*.name, 'test-flaky-ci') && github.head_ref || github.run_id }}",
+      "${{ github.workflow }}-${{ !contains(github.event.pull_request.labels.*.name, 'ci-test-flaky') && github.head_ref || github.run_id }}",
     "cancel-in-progress": true,
   },
   jobs: {
@@ -243,6 +243,8 @@ const ci = {
               job: "bench",
               profile: "release",
               use_sysroot: true,
+              skip_pr:
+                "${{ !contains(github.event.pull_request.labels.*.name, 'ci-bench') }}",
             },
             {
               os: Runners.linux,
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 79590f0621..9fdf8f63ab 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ on:
       - synchronize
       - ready_for_review
 concurrency:
-  group: '${{ github.workflow }}-${{ !contains(github.event.pull_request.labels.*.name, ''test-flaky-ci'') && github.head_ref || github.run_id }}'
+  group: '${{ github.workflow }}-${{ !contains(github.event.pull_request.labels.*.name, ''ci-test-flaky'') && github.head_ref || github.run_id }}'
   cancel-in-progress: true
 jobs:
   build:
@@ -50,6 +50,7 @@ jobs:
             job: bench
             profile: release
             use_sysroot: true
+            skip_pr: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-bench'') }}'
           - os: '${{ github.repository == ''denoland/deno'' && ''ubuntu-20.04-xl'' || ''ubuntu-20.04'' }}'
             job: test
             profile: debug