diff --git a/.github/labeler.yml b/.github/labeler.yml
index 1a613e325..abb0d05aa 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -1,9 +1,3 @@
-needs changelog:
-- all:
-  - '!docs/changelog.rst'
-  any:
-  - '(include|pybind11|tools)/**/*'
-
 docs:
 - any:
   - 'docs/**/*.rst'
diff --git a/.github/labeler_merged.yml b/.github/labeler_merged.yml
new file mode 100644
index 000000000..2374ad42e
--- /dev/null
+++ b/.github/labeler_merged.yml
@@ -0,0 +1,3 @@
+needs changelog:
+- all:
+  - '!docs/changelog.rst'
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 845b1d5d4..8e833fc38 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -3,11 +3,16 @@ on:
   pull_request_target:
 
 jobs:
-  label-merged:
-    name: Changelog needed
+  label:
+    name: Labeler
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@main
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: true
+    - uses: actions/labeler@main
+      if: "!(github.event.action == 'closed' && github.event.pull_request.merged == false)"
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        configuration-path: .github/labeler_merged.yml