diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aca2c820c2..abf3259ccd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,15 +22,14 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-          labels: |
-            git-branch={{branch}}
       - name: Set up persistent Buildx builder
         id: buildx
         uses: docker/setup-buildx-action@v3
       - name: Build
         uses: docker/bake-action@v6
         env:
-          GIT_BRANCH: ${{ fromJSON(steps.meta.outputs.json).labels['git-branch'] }}
+          # latest if branch is main, otherwise image version which is the pull request number
+          BUILD_CACHE_KEY: "${{ github.ref == 'refs/heads/main' ? "latest" : fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}"
         with:
           source: .
           push: false
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 9c085c3848..4a25283ebe 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -4,7 +4,7 @@ variable "IMAGE_REGISTRY" {
   default = "ghcr.io/zitadel"
 }
 
-variable "GIT_BRANCH" {
+variable "BUILD_CACHE_KEY" {
   default = "local"
 }
 
@@ -28,10 +28,10 @@ target "login-lint" {
     login-dev-base = "target:login-dev-base"
   }
   cache-from = [
-    "type=registry,ref=${IMAGE_REGISTRY}/login-lint-buildcache:${GIT_BRANCH}",
+    "type=registry,ref=${IMAGE_REGISTRY}/login-lint-buildcache:${BUILD_CACHE_KEY}",
     "type=registry,ref=${IMAGE_REGISTRY}/login-lint-buildcache:main"
   ]
-  cache-to = ["type=registry,ref=${IMAGE_REGISTRY}/login-lint-buildcache:${GIT_BRANCH},mode=max"]
+  cache-to = ["type=registry,ref=${IMAGE_REGISTRY}/login-lint-buildcache:${BUILD_CACHE_KEY},mode=max"]
 }
 
 variable "LOGIN_TEST_UNIT_TAG" {