diff --git a/packages/vite/src/client.ts b/packages/vite/src/client.ts
index b007f1ce3d..2f2af64eac 100644
--- a/packages/vite/src/client.ts
+++ b/packages/vite/src/client.ts
@@ -10,7 +10,6 @@ import { joinURL, withoutLeadingSlash } from 'ufo'
 import { defu } from 'defu'
 import { defineEventHandler } from 'h3'
 import type { ViteConfig } from '@nuxt/schema'
-import { cacheDirPlugin } from './plugins/cache-dir'
 import { chunkErrorPlugin } from './plugins/chunk-error'
 import type { ViteBuildContext } from './vite'
 import { devStyleSSRPlugin } from './plugins/dev-ssr-css'
@@ -51,6 +50,7 @@ export async function buildClient (ctx: ViteBuildContext) {
       },
       dedupe: ['vue']
     },
+    cacheDir: resolve(ctx.nuxt.options.rootDir, 'node_modules/.cache/vite', 'client'),
     build: {
       sourcemap: ctx.nuxt.options.sourcemap.client ? ctx.config.build?.sourcemap ?? true : false,
       manifest: true,
@@ -60,7 +60,6 @@ export async function buildClient (ctx: ViteBuildContext) {
       }
     },
     plugins: [
-      cacheDirPlugin(ctx.nuxt.options.rootDir, 'client'),
       devStyleSSRPlugin({
         srcDir: ctx.nuxt.options.srcDir,
         buildAssetsURL: joinURL(ctx.nuxt.options.app.baseURL, ctx.nuxt.options.app.buildAssetsDir)
diff --git a/packages/vite/src/plugins/cache-dir.ts b/packages/vite/src/plugins/cache-dir.ts
deleted file mode 100644
index 8ee8f0df14..0000000000
--- a/packages/vite/src/plugins/cache-dir.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { resolve } from 'pathe'
-import type { Plugin } from 'vite'
-
-export function cacheDirPlugin (rootDir: string, name: string) {
-  const optimizeCacheDir = resolve(rootDir, 'node_modules/.cache/vite', name)
-  return <Plugin> {
-    name: 'nuxt:cache-dir',
-    configResolved (resolvedConfig) {
-      // @ts-expect-error
-      resolvedConfig.optimizeCacheDir = optimizeCacheDir
-    }
-  }
-}
diff --git a/packages/vite/src/server.ts b/packages/vite/src/server.ts
index 75237f5295..6cfaa0e54d 100644
--- a/packages/vite/src/server.ts
+++ b/packages/vite/src/server.ts
@@ -7,7 +7,6 @@ import { joinURL, withTrailingSlash, withoutLeadingSlash } from 'ufo'
 import type { ViteConfig } from '@nuxt/schema'
 import type { ViteBuildContext } from './vite'
 import { createViteLogger } from './utils/logger'
-import { cacheDirPlugin } from './plugins/cache-dir'
 import { initViteNodeServer } from './vite-node'
 import { ssrStylesPlugin } from './plugins/ssr-styles'
 import { pureAnnotationsPlugin } from './plugins/pure-annotations'
@@ -82,6 +81,7 @@ export async function buildServer (ctx: ViteBuildContext) {
         /(nuxt|nuxt3)\/(dist|src|app)/
       ]
     },
+    cacheDir: resolve(ctx.nuxt.options.rootDir, 'node_modules/.cache/vite', 'server'),
     build: {
       sourcemap: ctx.nuxt.options.sourcemap.server ? ctx.config.build?.sourcemap ?? true : false,
       outDir: resolve(ctx.nuxt.options.buildDir, 'dist/server'),
@@ -110,7 +110,6 @@ export async function buildServer (ctx: ViteBuildContext) {
       hmr: false
     },
     plugins: [
-      cacheDirPlugin(ctx.nuxt.options.rootDir, 'server'),
       pureAnnotationsPlugin.vite({
         sourcemap: ctx.nuxt.options.sourcemap.server,
         functions: ['defineComponent', 'defineAsyncComponent', 'defineNuxtLink', 'createClientOnly', 'defineNuxtPlugin', 'defineNuxtRouteMiddleware', 'defineNuxtComponent', 'useRuntimeConfig']