feat(nuxt): cache vue app build outputs (#28726)

docs/2.guide/3.going-further/1.experimental-features.md

@@ -359,3 +359,34 @@ export default defineNuxtConfig({
 ::read-more{icon="i-simple-icons-mdnwebdocs" color="gray" to="https://developer.mozilla.org/en-US/docs/Web/API/CookieStore" target="_blank"}
 Read more about the **CookieStore**.
 ::
+
+## buildCache
+
+Caches Nuxt build artifacts based on a hash of the configuration and source files.
+
+```ts twoslash [nuxt.config.ts]
+export default defineNuxtConfig({
+  experimental: {
+    buildCache: true
+  }
+})
+```
+
+When enabled, changes to the following files will trigger a full rebuild:
+
+```bash [Directory structure]
+.nuxtrc
+.npmrc
+package.json
+package-lock.json
+yarn.lock
+pnpm-lock.yaml
+tsconfig.json
+bun.lockb
+```
+
+In addition, any changes to files within `srcDir` will trigger a rebuild of the Vue client/server bundle. Nitro will always be rebuilt (though work is in progress to allow Nitro to announce its cacheable artifacts and their hashes).
+
+::note
+A maximum of 10 cache tarballs are kept.
+::

packages/kit/src/loader/config.ts

@@ -1,3 +1,4 @@
+import { existsSync } from 'node:fs'
 import type { JSValue } from 'untyped'
 import { applyDefaults } from 'untyped'
 import type { ConfigLayer, ConfigLayerMeta, LoadConfigOptions } from 'c12'
@@ -6,6 +7,7 @@ import type { NuxtConfig, NuxtOptions } from '@nuxt/schema'
 import { NuxtConfigSchema } from '@nuxt/schema'
 import { globby } from 'globby'
 import defu from 'defu'
+import { join } from 'pathe'
 
 export interface LoadNuxtConfigOptions extends Omit<LoadConfigOptions<NuxtConfig>, 'overrides'> {
   // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
@@ -47,6 +49,11 @@ export async function loadNuxtConfig (opts: LoadNuxtConfigOptions): Promise<Nuxt
   nuxtConfig._nuxtConfigFile = configFile
   nuxtConfig._nuxtConfigFiles = [configFile]
 
+  const defaultBuildDir = join(nuxtConfig.rootDir!, '.nuxt')
+  if (!opts.overrides?._prepare && !nuxtConfig.dev && !nuxtConfig.buildDir && existsSync(defaultBuildDir)) {
+    nuxtConfig.buildDir = join(nuxtConfig.rootDir!, 'node_modules/.cache/nuxt/.nuxt')
+  }
+
   const _layers: ConfigLayer<NuxtConfig, ConfigLayerMeta>[] = []
   const processedLayers = new Set<string>()
   for (const layer of layers) {

packages/nuxt/package.json

@@ -91,6 +91,7 @@
     "knitwork": "^1.1.0",
     "magic-string": "^0.30.11",
     "mlly": "^1.7.1",
+    "nanotar": "^0.1.1",
     "nitro": "npm:nitro-nightly@3.0.0-beta-28665895.e727afda",
     "nuxi": "^3.13.1",
     "nypm": "^0.3.11",
@@ -104,6 +105,7 @@
     "semver": "^7.6.3",
     "std-env": "^3.7.0",
     "strip-literal": "^2.1.0",
+    "tinyglobby": "0.2.5",
     "ufo": "^1.5.4",
     "ultrahtml": "^1.5.3",
     "uncrypto": "^0.1.3",

packages/nuxt/src/core/builder.ts

@@ -8,6 +8,7 @@ import type { Nuxt, NuxtBuilder } from 'nuxt/schema'
 
 import { generateApp as _generateApp, createApp } from './app'
 import { checkForExternalConfigurationFiles } from './external-config-files'
+import { cleanupCaches, getVueHash } from './cache'
 
 export async function build (nuxt: Nuxt) {
   const app = createApp(nuxt)
@@ -40,16 +41,32 @@ export async function build (nuxt: Nuxt) {
     })
   }
 
-  await nuxt.callHook('build:before')
-  if (!nuxt.options._prepare) {
-    await Promise.all([checkForExternalConfigurationFiles(), bundle(nuxt)])
-    await nuxt.callHook('build:done')
-
-    if (!nuxt.options.dev) {
-      await nuxt.callHook('close', nuxt)
+  if (!nuxt.options._prepare && !nuxt.options.dev && nuxt.options.experimental.buildCache) {
+    const { restoreCache, collectCache } = await getVueHash(nuxt)
+    if (await restoreCache()) {
+      await nuxt.callHook('build:done')
+      return await nuxt.callHook('close', nuxt)
     }
-  } else {
+    nuxt.hooks.hookOnce('nitro:build:before', () => collectCache())
+    nuxt.hooks.hookOnce('close', () => cleanupCaches(nuxt))
+  }
+
+  await nuxt.callHook('build:before')
+  if (nuxt.options._prepare) {
     nuxt.hook('prepare:types', () => nuxt.close())
+    return
+  }
+
+  if (nuxt.options.dev) {
+    checkForExternalConfigurationFiles()
+  }
+
+  await bundle(nuxt)
+
+  await nuxt.callHook('build:done')
+
+  if (!nuxt.options.dev) {
+    await nuxt.callHook('close', nuxt)
   }
 }
 

packages/nuxt/src/core/cache.ts

@@ -0,0 +1,275 @@
+import { mkdir, open, readFile, stat, unlink, writeFile } from 'node:fs/promises'
+import type { FileHandle } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { existsSync } from 'node:fs'
+import { isIgnored } from '@nuxt/kit'
+import type { Nuxt, NuxtConfig, NuxtConfigLayer } from '@nuxt/schema'
+import { hash, murmurHash, objectHash } from 'ohash'
+import { glob } from 'tinyglobby'
+import _consola, { consola } from 'consola'
+import { dirname, join, relative } from 'pathe'
+import { createTar, parseTar } from 'nanotar'
+import type { TarFileInput } from 'nanotar'
+
+export async function getVueHash (nuxt: Nuxt) {
+  const id = 'vue'
+
+  const { hash } = await getHashes(nuxt, {
+    id,
+    cwd: layer => layer.config?.srcDir,
+    patterns: layer => [
+      join(relative(layer.cwd, layer.config.srcDir), '**'),
+      `!${relative(layer.cwd, layer.config.serverDir || join(layer.cwd, 'server'))}/**`,
+      `!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.public || 'public'))}/**`,
+      `!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.static || 'public'))}/**`,
+      '!node_modules/**',
+      '!nuxt.config.*',
+    ],
+    configOverrides: {
+      buildId: undefined,
+      serverDir: undefined,
+      nitro: undefined,
+      devServer: undefined,
+      runtimeConfig: undefined,
+      logLevel: undefined,
+      devServerHandlers: undefined,
+      generate: undefined,
+      devtools: undefined,
+    },
+  })
+
+  const cacheFile = join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds', id, hash + '.tar')
+
+  return {
+    hash,
+    async collectCache () {
+      const start = Date.now()
+      await writeCache(nuxt.options.buildDir, nuxt.options.buildDir, cacheFile)
+      const elapsed = Date.now() - start
+      consola.success(`Cached Vue client and server builds in \`${elapsed}ms\`.`)
+    },
+    async restoreCache () {
+      const start = Date.now()
+      const res = await restoreCache(nuxt.options.buildDir, cacheFile)
+      const elapsed = Date.now() - start
+      if (res) {
+        consola.success(`Restored Vue client and server builds from cache in \`${elapsed}ms\`.`)
+      }
+      return res
+    },
+  }
+}
+
+export async function cleanupCaches (nuxt: Nuxt) {
+  const start = Date.now()
+  const caches = await glob(['*/*.tar'], {
+    cwd: join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds'),
+    absolute: true,
+  })
+  if (caches.length >= 10) {
+    const cachesWithMeta = await Promise.all(caches.map(async (cache) => {
+      return [cache, await stat(cache).then(r => r.mtime.getTime()).catch(() => 0)] as const
+    }))
+    cachesWithMeta.sort((a, b) => a[1] - b[1])
+    for (const [cache] of cachesWithMeta.slice(0, cachesWithMeta.length - 10)) {
+      await unlink(cache)
+    }
+    const elapsed = Date.now() - start
+    consola.success(`Cleaned up old build caches in \`${elapsed}ms\`.`)
+  }
+}
+
+// internal
+
+type HashSource = { name: string, data: any }
+type Hashes = { hash: string, sources: HashSource[] }
+
+interface GetHashOptions {
+  id: string
+  cwd: (layer: NuxtConfigLayer) => string
+  patterns: (layer: NuxtConfigLayer) => string[]
+  configOverrides: Partial<Record<keyof NuxtConfig, unknown>>
+}
+
+async function getHashes (nuxt: Nuxt, options: GetHashOptions): Promise<Hashes> {
+  if ((nuxt as any)[`_${options.id}BuildHash`]) {
+    return (nuxt as any)[`_${options.id}BuildHash`]
+  }
+
+  const start = Date.now()
+  const hashSources: HashSource[] = []
+
+  // Layers
+  let layerCtr = 0
+  for (const layer of nuxt.options._layers) {
+    if (layer.cwd.includes('node_modules')) { continue }
+
+    const layerName = `layer#${layerCtr++}`
+    hashSources.push({
+      name: `${layerName}:config`,
+      data: objectHash({
+        ...layer.config,
+        ...options.configOverrides || {},
+      }),
+    })
+
+    const normalizeFiles = (files: Awaited<ReturnType<typeof readFilesRecursive>>) => files.map(f => ({
+      name: f.name,
+      size: (f.attrs as any)?.size,
+      data: murmurHash(f.data as any /* ArrayBuffer */),
+    }))
+
+    const sourceFiles = await readFilesRecursive(options.cwd(layer), {
+      shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths
+      cwd: nuxt.options.rootDir,
+      patterns: options.patterns(layer),
+    })
+
+    hashSources.push({
+      name: `${layerName}:src`,
+      data: normalizeFiles(sourceFiles),
+    })
+
+    const rootFiles = await readFilesRecursive(layer.config?.rootDir || layer.cwd, {
+      shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths
+      cwd: nuxt.options.rootDir,
+      patterns: [
+        '.nuxtrc',
+        '.npmrc',
+        'package.json',
+        'package-lock.json',
+        'yarn.lock',
+        'pnpm-lock.yaml',
+        'tsconfig.json',
+        'bun.lockb',
+      ],
+    })
+
+    hashSources.push({
+      name: `${layerName}:root`,
+      data: normalizeFiles(rootFiles),
+    })
+  }
+
+  const res = ((nuxt as any)[`_${options.id}BuildHash`] = {
+    hash: hash(hashSources),
+    sources: hashSources,
+  })
+
+  const elapsed = Date.now() - start
+  consola.debug(`Computed \`${options.id}\` build hash in \`${elapsed}ms\`.`)
+
+  return res
+}
+
+type FileWithMeta = TarFileInput & {
+  attrs: {
+    mtime: number
+    size: number
+  }
+}
+
+interface ReadFilesRecursiveOptions {
+  shouldIgnore?: (name: string) => boolean
+  patterns: string[]
+  cwd: string
+}
+
+async function readFilesRecursive (dir: string | string[], opts: ReadFilesRecursiveOptions): Promise<FileWithMeta[]> {
+  if (Array.isArray(dir)) {
+    return (await Promise.all(dir.map(d => readFilesRecursive(d, opts)))).flat()
+  }
+
+  const files = await glob(opts.patterns, { cwd: dir })
+
+  const fileEntries = await Promise.all(files.map(async (fileName) => {
+    if (!opts.shouldIgnore?.(fileName)) {
+      const file = await readFileWithMeta(dir, fileName)
+      if (!file) { return }
+      return {
+        ...file,
+        name: relative(opts.cwd, join(dir, file.name)),
+      }
+    }
+  }))
+
+  return fileEntries.filter(Boolean) as FileWithMeta[]
+}
+
+async function readFileWithMeta (dir: string, fileName: string, count = 0): Promise<FileWithMeta | undefined> {
+  let fd: FileHandle | undefined = undefined
+
+  try {
+    fd = await open(resolve(dir, fileName))
+    const stats = await fd.stat()
+
+    if (!stats?.isFile()) { return }
+
+    const mtime = stats.mtime.getTime()
+    const data = await fd.readFile()
+
+    // retry if file has changed during read
+    if ((await fd.stat()).mtime.getTime() !== mtime) {
+      if (count < 5) {
+        return readFileWithMeta(dir, fileName, count + 1)
+      }
+      console.warn(`Failed to read file \`${fileName}\` as it changed during read.`)
+      return
+    }
+
+    return {
+      name: fileName,
+      data,
+      attrs: {
+        mtime,
+        size: stats.size,
+      },
+    }
+  } catch (err) {
+    console.warn(`Failed to read file \`${fileName}\`:`, err)
+  } finally {
+    await fd?.close()
+  }
+}
+
+async function restoreCache (cwd: string, cacheFile: string) {
+  if (!existsSync(cacheFile)) {
+    return false
+  }
+
+  const files = parseTar(await readFile(cacheFile))
+  for (const file of files) {
+    let fd: FileHandle | undefined = undefined
+    try {
+      const filePath = resolve(cwd, file.name)
+      await mkdir(dirname(filePath), { recursive: true })
+
+      fd = await open(filePath, 'w')
+
+      const stats = await fd.stat().catch(() => null)
+      if (stats?.isFile() && stats.size) {
+        const lastModified = Number.parseInt(file.attrs?.mtime?.toString().padEnd(13, '0') || '0')
+        if (stats.mtime.getTime() >= lastModified) {
+          consola.debug(`Skipping \`${file.name}\` (up to date or newer than cache)`)
+          continue
+        }
+      }
+      await fd.writeFile(file.data!)
+    } catch (err) {
+      console.error(err)
+    } finally {
+      await fd?.close()
+    }
+  }
+  return true
+}
+
+async function writeCache (cwd: string, sources: string | string[], cacheFile: string) {
+  const fileEntries = await readFilesRecursive(sources, {
+    patterns: ['**/*', '!analyze/**'],
+    cwd,
+  })
+  const tarData = createTar(fileEntries)
+  await mkdir(dirname(cacheFile), { recursive: true })
+  await writeFile(cacheFile, tarData)
+}

packages/nuxt/src/core/nitro.ts

@@ -517,26 +517,30 @@ export async function initNitro (nuxt: Nuxt & { _nitro?: Nitro }) {
     })
   }
 
+  async function symlinkDist () {
+    if (nitro.options.static) {
+      const distDir = resolve(nuxt.options.rootDir, 'dist')
+      if (!existsSync(distDir)) {
+        await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {})
+      }
+    }
+  }
+
   // nuxt build/dev
   nuxt.hook('build:done', async () => {
     await nuxt.callHook('nitro:build:before', nitro)
     if (nuxt.options.dev) {
-      await build(nitro)
-    } else {
-      await prepare(nitro)
-      await prerender(nitro)
-
-      logger.restoreAll()
-      await build(nitro)
-      logger.wrapAll()
-
-      if (nitro.options.static) {
-        const distDir = resolve(nuxt.options.rootDir, 'dist')
-        if (!existsSync(distDir)) {
-          await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {})
-        }
-      }
+      return build(nitro)
     }
+
+    await prepare(nitro)
+    await prerender(nitro)
+
+    logger.restoreAll()
+    await build(nitro)
+    logger.wrapAll()
+
+    await symlinkDist()
   })
 
   // nuxt dev

packages/nuxt/src/core/plugins/resolve-deep-imports.ts

@@ -15,11 +15,13 @@ export function resolveDeepImportsPlugin (nuxt: Nuxt): Plugin {
       if (!importer || isAbsolute(id) || (!isAbsolute(importer) && !importer.startsWith('virtual:')) || exclude.some(e => id.startsWith(e))) {
         return
       }
-      id = normalize(id)
-      id = resolveAlias(id, nuxt.options.alias)
-      const { dir } = parseNodeModulePath(importer)
-      return await this.resolve?.(id, dir || pkgDir, { skipSelf: true }) ?? await resolvePath(id, {
-        url: [dir || pkgDir, ...nuxt.options.modulesDir],
+
+      const normalisedId = resolveAlias(normalize(id), nuxt.options.alias)
+      const normalisedImporter = importer.replace(/^\0?virtual:(?:nuxt:)?/, '')
+      const dir = parseNodeModulePath(normalisedImporter).dir || pkgDir
+
+      return await this.resolve?.(normalisedId, dir, { skipSelf: true }) ?? await resolvePath(id, {
+        url: [dir, ...nuxt.options.modulesDir],
         // TODO: respect nitro runtime conditions
         conditions: options.ssr ? ['node', 'import', 'require'] : ['import', 'require'],
       }).catch(() => {

packages/schema/src/config/common.ts

@@ -178,28 +178,9 @@ export default defineUntypedSchema({
    * ```
    */
   buildDir: {
-    $resolve: async (val: string | undefined, get): Promise<string> => {
+    $resolve: async (val: string | undefined, get) => {
       const rootDir = await get('rootDir') as string
-
-      if (val) {
-        return resolve(rootDir, val)
-      }
-
-      const defaultBuildDir = resolve(rootDir, '.nuxt')
-
-      const isDev = await get('dev') as boolean
-      if (isDev) {
-        return defaultBuildDir
-      }
-
-      // TODO: nuxi CLI should ensure .nuxt dir exists
-      if (!existsSync(defaultBuildDir)) {
-        // This is to ensure that types continue to work for CI builds
-        return defaultBuildDir
-      }
-
-      // TODO: handle build caching + using buildId in directory
-      return resolve(rootDir, 'node_modules/.cache/nuxt/builds', 'production')
+      return resolve(rootDir, val ?? '.nuxt')
     },
   },
 

packages/schema/src/config/experimental.ts

@@ -382,5 +382,12 @@ export default defineUntypedSchema({
      * It can reduce INP when navigating on prerendered routes.
      */
     navigationRepaint: true,
+
+    /**
+     * Cache Nuxt/Nitro build artifacts based on a hash of the configuration and source files.
+     *
+     * This only works for source files within `srcDir` and `serverDir` for the Vue/Nitro parts of your app.
+     */
+    buildCache: false,
   },
 })

pnpm-lock.yaml

@@ -363,6 +363,9 @@ importers:
       mlly:
         specifier: ^1.7.1
         version: 1.7.1
+      nanotar:
+        specifier: ^0.1.1
+        version: 0.1.1
       nitro:
         specifier: npm:nitro-nightly@3.0.0-beta-28665895.e727afda
         version: nitro-nightly@3.0.0-beta-28665895.e727afda(@opentelemetry/api@1.9.0)(encoding@0.1.13)(typescript@5.5.4)
@@ -402,6 +405,9 @@ importers:
       strip-literal:
         specifier: ^2.1.0
         version: 2.1.0
+      tinyglobby:
+        specifier: 0.2.5
+        version: 0.2.5
       ufo:
         specifier: ^1.5.4
         version: 1.5.4
@@ -5463,6 +5469,9 @@ packages:
     engines: {node: ^18 || >=20}
     hasBin: true
 
+  nanotar@0.1.1:
+    resolution: {integrity: sha512-AiJsGsSF3O0havL1BydvI4+wR76sKT+okKRwWIaK96cZUnXqH0uNBOsHlbwZq3+m2BR1VKqHDVudl3gO4mYjpQ==}
+
   natural-compare-lite@1.4.0:
     resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==}
 
@@ -12747,6 +12756,8 @@ snapshots:
 
   nanoid@5.0.7: {}
 
+  nanotar@0.1.1: {}
+
   natural-compare-lite@1.4.0: {}
 
   natural-compare@1.4.0: {}