feat(vue-renderer): use async fs (#5186)

packages/server/src/server.js

@@ -228,6 +228,9 @@ export default class Server {
   }
 
   async listen(port, host, socket) {
+    // Don't start listening before nuxt is ready
+    await this.nuxt.ready()
+
     // Create a new listener
     const listener = new Listener({
       port: isNaN(parseInt(port)) ? this.options.server.port : port,

packages/server/test/server.test.js

@@ -57,6 +57,7 @@ describe('server: server', () => {
       serverMiddleware: []
     },
     hook: jest.fn(),
+    ready: jest.fn(),
     callHook: jest.fn(),
     resolver: {
       requireModule: jest.fn()

packages/vue-renderer/src/renderer.js

@@ -1,6 +1,6 @@
 import path from 'path'
 import crypto from 'crypto'
-import fs from 'fs'
+import fs from 'fs-extra'
 import consola from 'consola'
 import devalue from '@nuxt/devalue'
 import invert from 'lodash/invert'
@@ -113,7 +113,7 @@ export default class VueRenderer {
   async ready() {
     // -- Development mode --
     if (this.context.options.dev) {
-      this.context.nuxt.hook('build:resources', mfs => this.loadResources(mfs, true))
+      this.context.nuxt.hook('build:resources', mfs => this.loadResources(mfs))
       return
     }
 
@@ -141,34 +141,28 @@ export default class VueRenderer {
     }
   }
 
-  loadResources(_fs, isMFS = false) {
+  async loadResources(_fs) {
     const distPath = path.resolve(this.context.options.buildDir, 'dist', 'server')
     const updated = []
-    const { resourceMap } = this
 
-    const readResource = (fileName, encoding) => {
+    const readResource = async (fileName, encoding) => {
       try {
         const fullPath = path.resolve(distPath, fileName)
-        if (!_fs.existsSync(fullPath)) {
+        if (!await _fs.exists(fullPath)) {
           return
         }
-        const contents = _fs.readFileSync(fullPath, encoding)
+        const contents = await _fs.readFile(fullPath, encoding)
-        if (isMFS) {
-          // Cleanup MFS as soon as possible to save memory
-          _fs.unlinkSync(fullPath)
-          delete this._assetsMapping
-        }
         return contents
       } catch (err) {
         consola.error('Unable to load resource:', fileName, err)
       }
     }
 
-    for (const resourceName in resourceMap) {
+    for (const resourceName in this.resourceMap) {
-      const { fileName, transform, encoding } = resourceMap[resourceName]
+      const { fileName, transform, encoding } = this.resourceMap[resourceName]
 
       // Load resource
-      let resource = readResource(fileName, encoding)
+      let resource = await readResource(fileName, encoding)
 
       // Skip unavailable resources
       if (!resource) {
@@ -178,10 +172,7 @@ export default class VueRenderer {
 
       // Apply transforms
       if (typeof transform === 'function') {
-        resource = transform(resource, {
+        resource = await transform(resource, { readResource })
-          readResource,
-          oldValue: this.context.resources[resourceName]
-        })
       }
 
       // Update resource
@@ -189,26 +180,15 @@ export default class VueRenderer {
       updated.push(resourceName)
     }
 
-    // Reload error template
+    // Load templates
-    const errorTemplatePath = path.resolve(this.context.options.buildDir, 'views/error.html')
+    await this.loadTemplates()
-    if (fs.existsSync(errorTemplatePath)) {
-      this.context.resources.errorTemplate = this.parseTemplate(
-        fs.readFileSync(errorTemplatePath, 'utf8')
-      )
-    }
 
-    // Reload loading template
+    // Detect if any resource updated
-    const loadingHTMLPath = path.resolve(this.context.options.buildDir, 'loading.html')
-    if (fs.existsSync(loadingHTMLPath)) {
-      this.context.resources.loadingHTML = fs.readFileSync(loadingHTMLPath, 'utf8')
-      this.context.resources.loadingHTML = this.context.resources.loadingHTML
-        .replace(/\r|\n|[\t\s]{3,}/g, '')
-    } else {
-      this.context.resources.loadingHTML = ''
-    }
-
-    // Call createRenderer if any resource changed
     if (updated.length > 0) {
+      // Invalidate assetsMapping cache
+      delete this._assetsMapping
+
+      // Create new renderer
       this.createRenderer()
     }
 
@@ -217,6 +197,24 @@ export default class VueRenderer {
     return this.context.nuxt.callHook('render:resourcesLoaded', this.context.resources)
   }
 
+  async loadTemplates() {
+    // Reload error template
+    const errorTemplatePath = path.resolve(this.context.options.buildDir, 'views/error.html')
+    if (await fs.exists(errorTemplatePath)) {
+      const errorTemplate = await fs.readFile(errorTemplatePath, 'utf8')
+      this.context.resources.errorTemplate = this.parseTemplate(errorTemplate)
+    }
+
+    // Reload loading template
+    const loadingHTMLPath = path.resolve(this.context.options.buildDir, 'loading.html')
+    if (await fs.exists(loadingHTMLPath)) {
+      this.context.resources.loadingHTML = await fs.readFile(loadingHTMLPath, 'utf8')
+      this.context.resources.loadingHTML = this.context.resources.loadingHTML.replace(/\r|\n|[\t\s]{3,}/g, '')
+    } else {
+      this.context.resources.loadingHTML = ''
+    }
+  }
+
   // TODO: Remove in Nuxt 3
   get noSSR() { /* Backward compatibility */
     return this.context.options.render.ssr === false
@@ -470,22 +468,21 @@ export default class VueRenderer {
       serverManifest: {
         fileName: 'server.manifest.json',
         // BundleRenderer needs resolved contents
-        transform: (src, { readResource, oldValue = { files: {}, maps: {} } }) => {
+        transform: async (src, { readResource }) => {
           const serverManifest = JSON.parse(src)
 
-          const resolveAssets = (obj, oldObj) => {
+          const readResources = async (obj) => {
-            Object.keys(obj).forEach((name) => {
+            const _obj = {}
-              obj[name] = readResource(obj[name])
+            await Promise.all(Object.keys(obj).map(async (key) => {
-              // Try to reuse deleted MFS files if no new version exists
+              _obj[key] = await readResource(obj[key])
-              if (!obj[name]) {
+            }))
-                obj[name] = oldObj[name]
+            return _obj
-              }
-            })
-            return obj
           }
 
-          const files = resolveAssets(serverManifest.files, oldValue.files)
+          const [files, maps] = await Promise.all([
-          const maps = resolveAssets(serverManifest.maps, oldValue.maps)
+            readResources(serverManifest.files),
+            readResources(serverManifest.maps)
+          ])
 
           // Try to parse sourcemaps
           for (const map in maps) {

packages/webpack/src/builder.js

@@ -1,7 +1,6 @@
 import path from 'path'
 import pify from 'pify'
 import webpack from 'webpack'
-import MFS from 'memory-fs'
 import Glob from 'glob'
 import webpackDevMiddleware from 'webpack-dev-middleware'
 import webpackHotMiddleware from 'webpack-hot-middleware'
@@ -12,6 +11,7 @@ import {
   sequence,
   wrapArray
 } from '@nuxt/utils'
+import AsyncMFS from './utils/async-mfs'
 
 import { ClientConfig, ModernConfig, ServerConfig } from './config'
 import PerfLoader from './utils/perf-loader'
@@ -29,11 +29,7 @@ export class WebpackBundler {
 
     // Initialize shared MFS for dev
     if (this.buildContext.options.dev) {
-      this.mfs = new MFS()
+      this.mfs = new AsyncMFS()
-
-      // TODO: Enable when async FS required
-      // this.mfs.exists = function (...args) { return Promise.resolve(this.existsSync(...args)) }
-      // this.mfs.readFile = function (...args) { return Promise.resolve(this.readFileSync(...args)) }
     }
   }
 

packages/webpack/src/utils/async-mfs.js

@@ -0,0 +1,24 @@
+import MFS from 'memory-fs'
+export default class AsyncMFS extends MFS {}
+
+const syncRegex = /Sync$/
+
+const propsToPromisify = Object.getOwnPropertyNames(MFS.prototype).filter(n => syncRegex.test(n))
+
+for (const prop of propsToPromisify) {
+  const asyncProp = prop.replace(syncRegex, '')
+  const origAsync = AsyncMFS.prototype[asyncProp]
+
+  AsyncMFS.prototype[asyncProp] = function (...args) {
+    // Callback support for webpack
+    if (origAsync && args.length && typeof args[args.length - 1] === 'function') {
+      return origAsync.call(this, ...args)
+    }
+
+    try {
+      return Promise.resolve(MFS.prototype[prop].call(this, ...args))
+    } catch (error) {
+      return Promise.reject(error)
+    }
+  }
+}

test/unit/express.test.js

@@ -13,6 +13,7 @@ describe('express', () => {
   beforeAll(async () => {
     const config = await loadFixture('basic')
     nuxt = new Nuxt(config)
+    await nuxt.ready()
 
     port = await getPort()