feat(nuxt): cache vue app build outputs (#28726)

This commit is contained in:
Daniel Roe 2024-08-28 21:00:38 +01:00 committed by GitHub
parent 87dca6a01d
commit e367cc9c48
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 386 additions and 49 deletions

View File

@ -359,3 +359,34 @@ export default defineNuxtConfig({
::read-more{icon="i-simple-icons-mdnwebdocs" color="gray" to="https://developer.mozilla.org/en-US/docs/Web/API/CookieStore" target="_blank"}
Read more about the **CookieStore**.
::
## buildCache
Caches Nuxt build artifacts based on a hash of the configuration and source files.
```ts twoslash [nuxt.config.ts]
export default defineNuxtConfig({
experimental: {
buildCache: true
}
})
```
When enabled, changes to the following files will trigger a full rebuild:
```bash [Directory structure]
.nuxtrc
.npmrc
package.json
package-lock.json
yarn.lock
pnpm-lock.yaml
tsconfig.json
bun.lockb
```
In addition, any changes to files within `srcDir` will trigger a rebuild of the Vue client/server bundle. Nitro will always be rebuilt (though work is in progress to allow Nitro to announce its cacheable artifacts and their hashes).
::note
A maximum of 10 cache tarballs are kept.
::

View File

@ -1,3 +1,4 @@
import { existsSync } from 'node:fs'
import type { JSValue } from 'untyped'
import { applyDefaults } from 'untyped'
import type { ConfigLayer, ConfigLayerMeta, LoadConfigOptions } from 'c12'
@ -6,6 +7,7 @@ import type { NuxtConfig, NuxtOptions } from '@nuxt/schema'
import { NuxtConfigSchema } from '@nuxt/schema'
import { globby } from 'globby'
import defu from 'defu'
import { join } from 'pathe'
export interface LoadNuxtConfigOptions extends Omit<LoadConfigOptions<NuxtConfig>, 'overrides'> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
@ -47,6 +49,11 @@ export async function loadNuxtConfig (opts: LoadNuxtConfigOptions): Promise<Nuxt
nuxtConfig._nuxtConfigFile = configFile
nuxtConfig._nuxtConfigFiles = [configFile]
const defaultBuildDir = join(nuxtConfig.rootDir!, '.nuxt')
if (!opts.overrides?._prepare && !nuxtConfig.dev && !nuxtConfig.buildDir && existsSync(defaultBuildDir)) {
nuxtConfig.buildDir = join(nuxtConfig.rootDir!, 'node_modules/.cache/nuxt/.nuxt')
}
const _layers: ConfigLayer<NuxtConfig, ConfigLayerMeta>[] = []
const processedLayers = new Set<string>()
for (const layer of layers) {

View File

@ -91,6 +91,7 @@
"knitwork": "^1.1.0",
"magic-string": "^0.30.11",
"mlly": "^1.7.1",
"nanotar": "^0.1.1",
"nitro": "npm:nitro-nightly@3.0.0-beta-28665895.e727afda",
"nuxi": "^3.13.1",
"nypm": "^0.3.11",
@ -104,6 +105,7 @@
"semver": "^7.6.3",
"std-env": "^3.7.0",
"strip-literal": "^2.1.0",
"tinyglobby": "0.2.5",
"ufo": "^1.5.4",
"ultrahtml": "^1.5.3",
"uncrypto": "^0.1.3",

View File

@ -8,6 +8,7 @@ import type { Nuxt, NuxtBuilder } from 'nuxt/schema'
import { generateApp as _generateApp, createApp } from './app'
import { checkForExternalConfigurationFiles } from './external-config-files'
import { cleanupCaches, getVueHash } from './cache'
export async function build (nuxt: Nuxt) {
const app = createApp(nuxt)
@ -40,16 +41,32 @@ export async function build (nuxt: Nuxt) {
})
}
await nuxt.callHook('build:before')
if (!nuxt.options._prepare) {
await Promise.all([checkForExternalConfigurationFiles(), bundle(nuxt)])
await nuxt.callHook('build:done')
if (!nuxt.options.dev) {
await nuxt.callHook('close', nuxt)
if (!nuxt.options._prepare && !nuxt.options.dev && nuxt.options.experimental.buildCache) {
const { restoreCache, collectCache } = await getVueHash(nuxt)
if (await restoreCache()) {
await nuxt.callHook('build:done')
return await nuxt.callHook('close', nuxt)
}
} else {
nuxt.hooks.hookOnce('nitro:build:before', () => collectCache())
nuxt.hooks.hookOnce('close', () => cleanupCaches(nuxt))
}
await nuxt.callHook('build:before')
if (nuxt.options._prepare) {
nuxt.hook('prepare:types', () => nuxt.close())
return
}
if (nuxt.options.dev) {
checkForExternalConfigurationFiles()
}
await bundle(nuxt)
await nuxt.callHook('build:done')
if (!nuxt.options.dev) {
await nuxt.callHook('close', nuxt)
}
}

View File

@ -0,0 +1,275 @@
import { mkdir, open, readFile, stat, unlink, writeFile } from 'node:fs/promises'
import type { FileHandle } from 'node:fs/promises'
import { resolve } from 'node:path'
import { existsSync } from 'node:fs'
import { isIgnored } from '@nuxt/kit'
import type { Nuxt, NuxtConfig, NuxtConfigLayer } from '@nuxt/schema'
import { hash, murmurHash, objectHash } from 'ohash'
import { glob } from 'tinyglobby'
import _consola, { consola } from 'consola'
import { dirname, join, relative } from 'pathe'
import { createTar, parseTar } from 'nanotar'
import type { TarFileInput } from 'nanotar'
export async function getVueHash (nuxt: Nuxt) {
const id = 'vue'
const { hash } = await getHashes(nuxt, {
id,
cwd: layer => layer.config?.srcDir,
patterns: layer => [
join(relative(layer.cwd, layer.config.srcDir), '**'),
`!${relative(layer.cwd, layer.config.serverDir || join(layer.cwd, 'server'))}/**`,
`!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.public || 'public'))}/**`,
`!${relative(layer.cwd, resolve(layer.config.srcDir || layer.cwd, layer.config.dir?.static || 'public'))}/**`,
'!node_modules/**',
'!nuxt.config.*',
],
configOverrides: {
buildId: undefined,
serverDir: undefined,
nitro: undefined,
devServer: undefined,
runtimeConfig: undefined,
logLevel: undefined,
devServerHandlers: undefined,
generate: undefined,
devtools: undefined,
},
})
const cacheFile = join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds', id, hash + '.tar')
return {
hash,
async collectCache () {
const start = Date.now()
await writeCache(nuxt.options.buildDir, nuxt.options.buildDir, cacheFile)
const elapsed = Date.now() - start
consola.success(`Cached Vue client and server builds in \`${elapsed}ms\`.`)
},
async restoreCache () {
const start = Date.now()
const res = await restoreCache(nuxt.options.buildDir, cacheFile)
const elapsed = Date.now() - start
if (res) {
consola.success(`Restored Vue client and server builds from cache in \`${elapsed}ms\`.`)
}
return res
},
}
}
export async function cleanupCaches (nuxt: Nuxt) {
const start = Date.now()
const caches = await glob(['*/*.tar'], {
cwd: join(nuxt.options.workspaceDir, 'node_modules/.cache/nuxt/builds'),
absolute: true,
})
if (caches.length >= 10) {
const cachesWithMeta = await Promise.all(caches.map(async (cache) => {
return [cache, await stat(cache).then(r => r.mtime.getTime()).catch(() => 0)] as const
}))
cachesWithMeta.sort((a, b) => a[1] - b[1])
for (const [cache] of cachesWithMeta.slice(0, cachesWithMeta.length - 10)) {
await unlink(cache)
}
const elapsed = Date.now() - start
consola.success(`Cleaned up old build caches in \`${elapsed}ms\`.`)
}
}
// internal
type HashSource = { name: string, data: any }
type Hashes = { hash: string, sources: HashSource[] }
interface GetHashOptions {
id: string
cwd: (layer: NuxtConfigLayer) => string
patterns: (layer: NuxtConfigLayer) => string[]
configOverrides: Partial<Record<keyof NuxtConfig, unknown>>
}
async function getHashes (nuxt: Nuxt, options: GetHashOptions): Promise<Hashes> {
if ((nuxt as any)[`_${options.id}BuildHash`]) {
return (nuxt as any)[`_${options.id}BuildHash`]
}
const start = Date.now()
const hashSources: HashSource[] = []
// Layers
let layerCtr = 0
for (const layer of nuxt.options._layers) {
if (layer.cwd.includes('node_modules')) { continue }
const layerName = `layer#${layerCtr++}`
hashSources.push({
name: `${layerName}:config`,
data: objectHash({
...layer.config,
...options.configOverrides || {},
}),
})
const normalizeFiles = (files: Awaited<ReturnType<typeof readFilesRecursive>>) => files.map(f => ({
name: f.name,
size: (f.attrs as any)?.size,
data: murmurHash(f.data as any /* ArrayBuffer */),
}))
const sourceFiles = await readFilesRecursive(options.cwd(layer), {
shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths
cwd: nuxt.options.rootDir,
patterns: options.patterns(layer),
})
hashSources.push({
name: `${layerName}:src`,
data: normalizeFiles(sourceFiles),
})
const rootFiles = await readFilesRecursive(layer.config?.rootDir || layer.cwd, {
shouldIgnore: isIgnored, // TODO: Validate if works with absolute paths
cwd: nuxt.options.rootDir,
patterns: [
'.nuxtrc',
'.npmrc',
'package.json',
'package-lock.json',
'yarn.lock',
'pnpm-lock.yaml',
'tsconfig.json',
'bun.lockb',
],
})
hashSources.push({
name: `${layerName}:root`,
data: normalizeFiles(rootFiles),
})
}
const res = ((nuxt as any)[`_${options.id}BuildHash`] = {
hash: hash(hashSources),
sources: hashSources,
})
const elapsed = Date.now() - start
consola.debug(`Computed \`${options.id}\` build hash in \`${elapsed}ms\`.`)
return res
}
type FileWithMeta = TarFileInput & {
attrs: {
mtime: number
size: number
}
}
interface ReadFilesRecursiveOptions {
shouldIgnore?: (name: string) => boolean
patterns: string[]
cwd: string
}
async function readFilesRecursive (dir: string | string[], opts: ReadFilesRecursiveOptions): Promise<FileWithMeta[]> {
if (Array.isArray(dir)) {
return (await Promise.all(dir.map(d => readFilesRecursive(d, opts)))).flat()
}
const files = await glob(opts.patterns, { cwd: dir })
const fileEntries = await Promise.all(files.map(async (fileName) => {
if (!opts.shouldIgnore?.(fileName)) {
const file = await readFileWithMeta(dir, fileName)
if (!file) { return }
return {
...file,
name: relative(opts.cwd, join(dir, file.name)),
}
}
}))
return fileEntries.filter(Boolean) as FileWithMeta[]
}
async function readFileWithMeta (dir: string, fileName: string, count = 0): Promise<FileWithMeta | undefined> {
let fd: FileHandle | undefined = undefined
try {
fd = await open(resolve(dir, fileName))
const stats = await fd.stat()
if (!stats?.isFile()) { return }
const mtime = stats.mtime.getTime()
const data = await fd.readFile()
// retry if file has changed during read
if ((await fd.stat()).mtime.getTime() !== mtime) {
if (count < 5) {
return readFileWithMeta(dir, fileName, count + 1)
}
console.warn(`Failed to read file \`${fileName}\` as it changed during read.`)
return
}
return {
name: fileName,
data,
attrs: {
mtime,
size: stats.size,
},
}
} catch (err) {
console.warn(`Failed to read file \`${fileName}\`:`, err)
} finally {
await fd?.close()
}
}
async function restoreCache (cwd: string, cacheFile: string) {
if (!existsSync(cacheFile)) {
return false
}
const files = parseTar(await readFile(cacheFile))
for (const file of files) {
let fd: FileHandle | undefined = undefined
try {
const filePath = resolve(cwd, file.name)
await mkdir(dirname(filePath), { recursive: true })
fd = await open(filePath, 'w')
const stats = await fd.stat().catch(() => null)
if (stats?.isFile() && stats.size) {
const lastModified = Number.parseInt(file.attrs?.mtime?.toString().padEnd(13, '0') || '0')
if (stats.mtime.getTime() >= lastModified) {
consola.debug(`Skipping \`${file.name}\` (up to date or newer than cache)`)
continue
}
}
await fd.writeFile(file.data!)
} catch (err) {
console.error(err)
} finally {
await fd?.close()
}
}
return true
}
async function writeCache (cwd: string, sources: string | string[], cacheFile: string) {
const fileEntries = await readFilesRecursive(sources, {
patterns: ['**/*', '!analyze/**'],
cwd,
})
const tarData = createTar(fileEntries)
await mkdir(dirname(cacheFile), { recursive: true })
await writeFile(cacheFile, tarData)
}

View File

@ -517,26 +517,30 @@ export async function initNitro (nuxt: Nuxt & { _nitro?: Nitro }) {
})
}
async function symlinkDist () {
if (nitro.options.static) {
const distDir = resolve(nuxt.options.rootDir, 'dist')
if (!existsSync(distDir)) {
await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {})
}
}
}
// nuxt build/dev
nuxt.hook('build:done', async () => {
await nuxt.callHook('nitro:build:before', nitro)
if (nuxt.options.dev) {
await build(nitro)
} else {
await prepare(nitro)
await prerender(nitro)
logger.restoreAll()
await build(nitro)
logger.wrapAll()
if (nitro.options.static) {
const distDir = resolve(nuxt.options.rootDir, 'dist')
if (!existsSync(distDir)) {
await fsp.symlink(nitro.options.output.publicDir, distDir, 'junction').catch(() => {})
}
}
return build(nitro)
}
await prepare(nitro)
await prerender(nitro)
logger.restoreAll()
await build(nitro)
logger.wrapAll()
await symlinkDist()
})
// nuxt dev

View File

@ -15,11 +15,13 @@ export function resolveDeepImportsPlugin (nuxt: Nuxt): Plugin {
if (!importer || isAbsolute(id) || (!isAbsolute(importer) && !importer.startsWith('virtual:')) || exclude.some(e => id.startsWith(e))) {
return
}
id = normalize(id)
id = resolveAlias(id, nuxt.options.alias)
const { dir } = parseNodeModulePath(importer)
return await this.resolve?.(id, dir || pkgDir, { skipSelf: true }) ?? await resolvePath(id, {
url: [dir || pkgDir, ...nuxt.options.modulesDir],
const normalisedId = resolveAlias(normalize(id), nuxt.options.alias)
const normalisedImporter = importer.replace(/^\0?virtual:(?:nuxt:)?/, '')
const dir = parseNodeModulePath(normalisedImporter).dir || pkgDir
return await this.resolve?.(normalisedId, dir, { skipSelf: true }) ?? await resolvePath(id, {
url: [dir, ...nuxt.options.modulesDir],
// TODO: respect nitro runtime conditions
conditions: options.ssr ? ['node', 'import', 'require'] : ['import', 'require'],
}).catch(() => {

View File

@ -178,28 +178,9 @@ export default defineUntypedSchema({
* ```
*/
buildDir: {
$resolve: async (val: string | undefined, get): Promise<string> => {
$resolve: async (val: string | undefined, get) => {
const rootDir = await get('rootDir') as string
if (val) {
return resolve(rootDir, val)
}
const defaultBuildDir = resolve(rootDir, '.nuxt')
const isDev = await get('dev') as boolean
if (isDev) {
return defaultBuildDir
}
// TODO: nuxi CLI should ensure .nuxt dir exists
if (!existsSync(defaultBuildDir)) {
// This is to ensure that types continue to work for CI builds
return defaultBuildDir
}
// TODO: handle build caching + using buildId in directory
return resolve(rootDir, 'node_modules/.cache/nuxt/builds', 'production')
return resolve(rootDir, val ?? '.nuxt')
},
},

View File

@ -382,5 +382,12 @@ export default defineUntypedSchema({
* It can reduce INP when navigating on prerendered routes.
*/
navigationRepaint: true,
/**
* Cache Nuxt/Nitro build artifacts based on a hash of the configuration and source files.
*
* This only works for source files within `srcDir` and `serverDir` for the Vue/Nitro parts of your app.
*/
buildCache: false,
},
})

View File

@ -363,6 +363,9 @@ importers:
mlly:
specifier: ^1.7.1
version: 1.7.1
nanotar:
specifier: ^0.1.1
version: 0.1.1
nitro:
specifier: npm:nitro-nightly@3.0.0-beta-28665895.e727afda
version: nitro-nightly@3.0.0-beta-28665895.e727afda(@opentelemetry/api@1.9.0)(encoding@0.1.13)(typescript@5.5.4)
@ -402,6 +405,9 @@ importers:
strip-literal:
specifier: ^2.1.0
version: 2.1.0
tinyglobby:
specifier: 0.2.5
version: 0.2.5
ufo:
specifier: ^1.5.4
version: 1.5.4
@ -5463,6 +5469,9 @@ packages:
engines: {node: ^18 || >=20}
hasBin: true
nanotar@0.1.1:
resolution: {integrity: sha512-AiJsGsSF3O0havL1BydvI4+wR76sKT+okKRwWIaK96cZUnXqH0uNBOsHlbwZq3+m2BR1VKqHDVudl3gO4mYjpQ==}
natural-compare-lite@1.4.0:
resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==}
@ -12747,6 +12756,8 @@ snapshots:
nanoid@5.0.7: {}
nanotar@0.1.1: {}
natural-compare-lite@1.4.0: {}
natural-compare@1.4.0: {}