diff --git a/docs/2.guide/3.going-further/1.experimental-features.md b/docs/2.guide/3.going-further/1.experimental-features.md index fd9c065baf..136cadf805 100644 --- a/docs/2.guide/3.going-further/1.experimental-features.md +++ b/docs/2.guide/3.going-further/1.experimental-features.md @@ -359,3 +359,34 @@ export default defineNuxtConfig({ ::read-more{icon="i-simple-icons-mdnwebdocs" color="gray" to="https://developer.mozilla.org/en-US/docs/Web/API/CookieStore" target="_blank"} Read more about the **CookieStore**. :: + +## buildCache + +Caches Nuxt build artifacts based on a hash of the configuration and source files. + +```ts twoslash [nuxt.config.ts] +export default defineNuxtConfig({ + experimental: { + buildCache: true + } +}) +``` + +When enabled, changes to the following files will trigger a full rebuild: + +```bash [Directory structure] +.nuxtrc +.npmrc +package.json +package-lock.json +yarn.lock +pnpm-lock.yaml +tsconfig.json +bun.lockb +``` + +In addition, any changes to files within `srcDir` will trigger a rebuild of the Vue client/server bundle. Nitro will always be rebuilt (though work is in progress to allow Nitro to announce its cacheable artifacts and their hashes). + +::note +A maximum of 10 cache tarballs are kept. +:: diff --git a/packages/kit/src/loader/config.ts b/packages/kit/src/loader/config.ts index 444bf8c46c..80139fb015 100644 --- a/packages/kit/src/loader/config.ts +++ b/packages/kit/src/loader/config.ts @@ -1,3 +1,4 @@ +import { existsSync } from 'node:fs' import type { JSValue } from 'untyped' import { applyDefaults } from 'untyped' import type { ConfigLayer, ConfigLayerMeta, LoadConfigOptions } from 'c12' @@ -6,6 +7,7 @@ import type { NuxtConfig, NuxtOptions } from '@nuxt/schema' import { NuxtConfigSchema } from '@nuxt/schema' import { globby } from 'globby' import defu from 'defu' +import { join } from 'pathe' export interface LoadNuxtConfigOptions extends Omit, 'overrides'> { // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type @@ -47,6 +49,11 @@ export async function loadNuxtConfig (opts: LoadNuxtConfigOptions): Promise[] = [] const processedLayers = new Set() for (const layer of layers) { diff --git a/packages/nuxt/package.json b/packages/nuxt/package.json index 60c0451c7a..b123572bd4 100644 --- a/packages/nuxt/package.json +++ b/packages/nuxt/package.json @@ -91,6 +91,7 @@ "knitwork": "^1.1.0", "magic-string": "^0.30.11", "mlly": "^1.7.1", + "nanotar": "^0.1.1", "nitro": "npm:nitro-nightly@3.0.0-beta-28665895.e727afda", "nuxi": "^3.13.1", "nypm": "^0.3.11", @@ -104,6 +105,7 @@ "semver": "^7.6.3", "std-env": "^3.7.0", "strip-literal": "^2.1.0", + "tinyglobby": "0.2.5", "ufo": "^1.5.4", "ultrahtml": "^1.5.3", "uncrypto": "^0.1.3", diff --git a/packages/nuxt/src/core/builder.ts b/packages/nuxt/src/core/builder.ts index 92bd04270d..b414cbf081 100644 --- a/packages/nuxt/src/core/builder.ts +++ b/packages/nuxt/src/core/builder.ts @@ -8,6 +8,7 @@ import type { Nuxt, NuxtBuilder } from 'nuxt/schema' import { generateApp as _generateApp, createApp } from './app' import { checkForExternalConfigurationFiles } from './external-config-files' +import { cleanupCaches, getVueHash } from './cache' export async function build (nuxt: Nuxt) { const app = createApp(nuxt) @@ -40,16 +41,32 @@ export async function build (nuxt: Nuxt) { }) } - await nuxt.callHook('build:before') - if (!nuxt.options._prepare) { - await Promise.all([checkForExternalConfigurationFiles(), bundle(nuxt)]) - await nuxt.callHook('build:done') - - if (!nuxt.options.dev) { - await nuxt.callHook('close', nuxt) + if (!nuxt.options._prepare && !nuxt.options.dev && nuxt.options.experimental.buildCache) { + const { restoreCache, collectCache } = await getVueHash(nuxt) + if (await restoreCache()) { + await nuxt.callHook('build:done') + return await nuxt.callHook('close', nuxt) } - } else { + nuxt.hooks.hookOnce('nitro:build:before', () => collectCache()) + nuxt.hooks.hookOnce('close', () => cleanupCaches(nuxt)) + } + + await nuxt.callHook('build:before') + if (nuxt.options._prepare) { nuxt.hook('prepare:types', () => nuxt.close()) + return + } + + if (nuxt.options.dev) { + checkForExternalConfigurationFiles() + } + + await bundle(nuxt) + + await nuxt.callHook('build:done') + + if (!nuxt.options.dev) { + await nuxt.callHook('close', nuxt) } } diff --git a/packages/nuxt/src/core/cache.ts b/packages/nuxt/src/core/cache.ts new file mode 100644 index 0000000000..748d57bc79 --- /dev/null +++ b/packages/nuxt/src/core/cache.ts @@ -0,0 +1,275 @@ +import { mkdir, open, readFile, stat, unlink, writeFile } from 'node:fs/promises' +import type { FileHandle } from 'node:fs/promises' +import { resolve } from 'node:path' +import { existsSync } from 'node:fs' +import { isIgnored } from '@nuxt/kit' +import type { Nuxt, NuxtConfig, NuxtConfigLayer } from '@nuxt/schema' +import { hash, murmurHash, objectHash } from 'ohash' +import { glob } from 'tinyglobby' +import _consola, { consola } from 'consola' +import { dirname, join, relative } from 'pathe' +import { createTar, parseTar } from 'nanotar' +import type { TarFileInput } from 'nanotar' + +export async function getVueHash (nuxt: Nuxt) { + const id = 'vue' + + const { hash } = await getHashes(nuxt, { + id, + cwd: layer => layer.config?.srcDir, + patterns: layer => [ + join(relative(layer.cwd, layer.config.srcDir), '**'), + `!${relative(layer.cwd, layer.config.serverDir || join(layer.cwd, 'server'))}/**`, + `!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.public || 'public'))}/**`, + `!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.static || 'public'))}/**`, + '!node_modules/**', + '!nuxt.config.*', + ], + configOverrides: { + buildId: undefined, + serverDir: undefined, + nitro: undefined, + devServer: undefined, + runtimeConfig: undefined, + logLevel: undefined, + devServerHandlers: undefined, + generate: undefined, + devtools: undefined, + }, + }) + + const cacheFile = join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds', id, hash + '.tar') + + return { + hash, + async collectCache () { + const start = Date.now() + await writeCache(nuxt.options.buildDir, nuxt.options.buildDir, cacheFile) + const elapsed = Date.now() - start + consola.success(`Cached Vue client and server builds in \`${elapsed}ms\`.`) + }, + async restoreCache () { + const start = Date.now() + const res = await restoreCache(nuxt.options.buildDir, cacheFile) + const elapsed = Date.now() - start + if (res) { + consola.success(`Restored Vue client and server builds from cache in \`${elapsed}ms\`.`) + } + return res + }, + } +} + +export async function cleanupCaches (nuxt: Nuxt) { + const start = Date.now() + const caches = await glob(['*/*.tar'], { + cwd: join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds'), + absolute: true, + }) + if (caches.length >= 10) { + const cachesWithMeta = await Promise.all(caches.map(async (cache) => { + return [cache, await stat(cache).then(r => r.mtime.getTime()).catch(() => 0)] as const + })) + cachesWithMeta.sort((a, b) => a[1] - b[1]) + for (const [cache] of cachesWithMeta.slice(0, cachesWithMeta.length - 10)) { + await unlink(cache) + } + const elapsed = Date.now() - start + consola.success(`Cleaned up old build caches in \`${elapsed}ms\`.`) + } +} + +// internal + +type HashSource = { name: string, data: any } +type Hashes = { hash: string, sources: HashSource[] } + +interface GetHashOptions { + id: string + cwd: (layer: NuxtConfigLayer) => string + patterns: (layer: NuxtConfigLayer) => string[] + configOverrides: Partial> +} + +async function getHashes (nuxt: Nuxt, options: GetHashOptions): Promise { + if ((nuxt as any)[`_${options.id}BuildHash`]) { + return (nuxt as any)[`_${options.id}BuildHash`] + } + + const start = Date.now() + const hashSources: HashSource[] = [] + + // Layers + let layerCtr = 0 + for (const layer of nuxt.options._layers) { + if (layer.cwd.includes('node_modules')) { continue } + + const layerName = `layer#${layerCtr++}` + hashSources.push({ + name: `${layerName}:config`, + data: objectHash({ + ...layer.config, + ...options.configOverrides || {}, + }), + }) + + const normalizeFiles = (files: Awaited>) => files.map(f => ({ + name: f.name, + size: (f.attrs as any)?.size, + data: murmurHash(f.data as any /* ArrayBuffer */), + })) + + const sourceFiles = await readFilesRecursive(options.cwd(layer), { + shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths + cwd: nuxt.options.rootDir, + patterns: options.patterns(layer), + }) + + hashSources.push({ + name: `${layerName}:src`, + data: normalizeFiles(sourceFiles), + }) + + const rootFiles = await readFilesRecursive(layer.config?.rootDir || layer.cwd, { + shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths + cwd: nuxt.options.rootDir, + patterns: [ + '.nuxtrc', + '.npmrc', + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'tsconfig.json', + 'bun.lockb', + ], + }) + + hashSources.push({ + name: `${layerName}:root`, + data: normalizeFiles(rootFiles), + }) + } + + const res = ((nuxt as any)[`_${options.id}BuildHash`] = { + hash: hash(hashSources), + sources: hashSources, + }) + + const elapsed = Date.now() - start + consola.debug(`Computed \`${options.id}\` build hash in \`${elapsed}ms\`.`) + + return res +} + +type FileWithMeta = TarFileInput & { + attrs: { + mtime: number + size: number + } +} + +interface ReadFilesRecursiveOptions { + shouldIgnore?: (name: string) => boolean + patterns: string[] + cwd: string +} + +async function readFilesRecursive (dir: string | string[], opts: ReadFilesRecursiveOptions): Promise { + if (Array.isArray(dir)) { + return (await Promise.all(dir.map(d => readFilesRecursive(d, opts)))).flat() + } + + const files = await glob(opts.patterns, { cwd: dir }) + + const fileEntries = await Promise.all(files.map(async (fileName) => { + if (!opts.shouldIgnore?.(fileName)) { + const file = await readFileWithMeta(dir, fileName) + if (!file) { return } + return { + ...file, + name: relative(opts.cwd, join(dir, file.name)), + } + } + })) + + return fileEntries.filter(Boolean) as FileWithMeta[] +} + +async function readFileWithMeta (dir: string, fileName: string, count = 0): Promise { + let fd: FileHandle | undefined = undefined + + try { + fd = await open(resolve(dir, fileName)) + const stats = await fd.stat() + + if (!stats?.isFile()) { return } + + const mtime = stats.mtime.getTime() + const data = await fd.readFile() + + // retry if file has changed during read + if ((await fd.stat()).mtime.getTime() !== mtime) { + if (count < 5) { + return readFileWithMeta(dir, fileName, count + 1) + } + console.warn(`Failed to read file \`${fileName}\` as it changed during read.`) + return + } + + return { + name: fileName, + data, + attrs: { + mtime, + size: stats.size, + }, + } + } catch (err) { + console.warn(`Failed to read file \`${fileName}\`:`, err) + } finally { + await fd?.close() + } +} + +async function restoreCache (cwd: string, cacheFile: string) { + if (!existsSync(cacheFile)) { + return false + } + + const files = parseTar(await readFile(cacheFile)) + for (const file of files) { + let fd: FileHandle | undefined = undefined + try { + const filePath = resolve(cwd, file.name) + await mkdir(dirname(filePath), { recursive: true }) + + fd = await open(filePath, 'w') + + const stats = await fd.stat().catch(() => null) + if (stats?.isFile() && stats.size) { + const lastModified = Number.parseInt(file.attrs?.mtime?.toString().padEnd(13, '0') || '0') + if (stats.mtime.getTime() >= lastModified) { + consola.debug(`Skipping \`${file.name}\` (up to date or newer than cache)`) + continue + } + } + await fd.writeFile(file.data!) + } catch (err) { + console.error(err) + } finally { + await fd?.close() + } + } + return true +} + +async function writeCache (cwd: string, sources: string | string[], cacheFile: string) { + const fileEntries = await readFilesRecursive(sources, { + patterns: ['**/*', '!analyze/**'], + cwd, + }) + const tarData = createTar(fileEntries) + await mkdir(dirname(cacheFile), { recursive: true }) + await writeFile(cacheFile, tarData) +} diff --git a/packages/nuxt/src/core/nitro.ts b/packages/nuxt/src/core/nitro.ts index c8ed9e5bbb..247a457e66 100644 --- a/packages/nuxt/src/core/nitro.ts +++ b/packages/nuxt/src/core/nitro.ts @@ -517,26 +517,30 @@ export async function initNitro (nuxt: Nuxt & { _nitro?: Nitro }) { }) } + async function symlinkDist () { + if (nitro.options.static) { + const distDir = resolve(nuxt.options.rootDir, 'dist') + if (!existsSync(distDir)) { + await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {}) + } + } + } + // nuxt build/dev nuxt.hook('build:done', async () => { await nuxt.callHook('nitro:build:before', nitro) if (nuxt.options.dev) { - await build(nitro) - } else { - await prepare(nitro) - await prerender(nitro) - - logger.restoreAll() - await build(nitro) - logger.wrapAll() - - if (nitro.options.static) { - const distDir = resolve(nuxt.options.rootDir, 'dist') - if (!existsSync(distDir)) { - await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {}) - } - } + return build(nitro) } + + await prepare(nitro) + await prerender(nitro) + + logger.restoreAll() + await build(nitro) + logger.wrapAll() + + await symlinkDist() }) // nuxt dev diff --git a/packages/nuxt/src/core/plugins/resolve-deep-imports.ts b/packages/nuxt/src/core/plugins/resolve-deep-imports.ts index b965141196..0eb0e205ec 100644 --- a/packages/nuxt/src/core/plugins/resolve-deep-imports.ts +++ b/packages/nuxt/src/core/plugins/resolve-deep-imports.ts @@ -15,11 +15,13 @@ export function resolveDeepImportsPlugin (nuxt: Nuxt): Plugin { if (!importer || isAbsolute(id) || (!isAbsolute(importer) && !importer.startsWith('virtual:')) || exclude.some(e => id.startsWith(e))) { return } - id = normalize(id) - id = resolveAlias(id, nuxt.options.alias) - const { dir } = parseNodeModulePath(importer) - return await this.resolve?.(id, dir || pkgDir, { skipSelf: true }) ?? await resolvePath(id, { - url: [dir || pkgDir, ...nuxt.options.modulesDir], + + const normalisedId = resolveAlias(normalize(id), nuxt.options.alias) + const normalisedImporter = importer.replace(/^\0?virtual:(?:nuxt:)?/, '') + const dir = parseNodeModulePath(normalisedImporter).dir || pkgDir + + return await this.resolve?.(normalisedId, dir, { skipSelf: true }) ?? await resolvePath(id, { + url: [dir, ...nuxt.options.modulesDir], // TODO: respect nitro runtime conditions conditions: options.ssr ? ['node', 'import', 'require'] : ['import', 'require'], }).catch(() => { diff --git a/packages/schema/src/config/common.ts b/packages/schema/src/config/common.ts index 1ec5350f22..aa668b95ed 100644 --- a/packages/schema/src/config/common.ts +++ b/packages/schema/src/config/common.ts @@ -178,28 +178,9 @@ export default defineUntypedSchema({ * ``` */ buildDir: { - $resolve: async (val: string | undefined, get): Promise => { + $resolve: async (val: string | undefined, get) => { const rootDir = await get('rootDir') as string - - if (val) { - return resolve(rootDir, val) - } - - const defaultBuildDir = resolve(rootDir, '.nuxt') - - const isDev = await get('dev') as boolean - if (isDev) { - return defaultBuildDir - } - - // TODO: nuxi CLI should ensure .nuxt dir exists - if (!existsSync(defaultBuildDir)) { - // This is to ensure that types continue to work for CI builds - return defaultBuildDir - } - - // TODO: handle build caching + using buildId in directory - return resolve(rootDir, 'node_modules/.cache/nuxt/builds', 'production') + return resolve(rootDir, val ?? '.nuxt') }, }, diff --git a/packages/schema/src/config/experimental.ts b/packages/schema/src/config/experimental.ts index 1e5aaf6b8a..8f0249cc3d 100644 --- a/packages/schema/src/config/experimental.ts +++ b/packages/schema/src/config/experimental.ts @@ -382,5 +382,12 @@ export default defineUntypedSchema({ * It can reduce INP when navigating on prerendered routes. */ navigationRepaint: true, + + /** + * Cache Nuxt/Nitro build artifacts based on a hash of the configuration and source files. + * + * This only works for source files within `srcDir` and `serverDir` for the Vue/Nitro parts of your app. + */ + buildCache: false, }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 239dec2952..d7082b24c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -363,6 +363,9 @@ importers: mlly: specifier: ^1.7.1 version: 1.7.1 + nanotar: + specifier: ^0.1.1 + version: 0.1.1 nitro: specifier: npm:nitro-nightly@3.0.0-beta-28665895.e727afda version: nitro-nightly@3.0.0-beta-28665895.e727afda(@opentelemetry/api@1.9.0)(encoding@0.1.13)(typescript@5.5.4) @@ -402,6 +405,9 @@ importers: strip-literal: specifier: ^2.1.0 version: 2.1.0 + tinyglobby: + specifier: 0.2.5 + version: 0.2.5 ufo: specifier: ^1.5.4 version: 1.5.4 @@ -5463,6 +5469,9 @@ packages: engines: {node: ^18 || >=20} hasBin: true + nanotar@0.1.1: + resolution: {integrity: sha512-AiJsGsSF3O0havL1BydvI4+wR76sKT+okKRwWIaK96cZUnXqH0uNBOsHlbwZq3+m2BR1VKqHDVudl3gO4mYjpQ==} + natural-compare-lite@1.4.0: resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==} @@ -12747,6 +12756,8 @@ snapshots: nanoid@5.0.7: {} + nanotar@0.1.1: {} + natural-compare-lite@1.4.0: {} natural-compare@1.4.0: {}