diff --git a/packages/webpack/src/builder.js b/packages/webpack/src/builder.js
index a23d8beb65..db6b4cc147 100644
--- a/packages/webpack/src/builder.js
+++ b/packages/webpack/src/builder.js
@@ -26,7 +26,6 @@ export class WebpackBundler {
     this.compilersWatching = []
     this.devMiddleware = {}
     this.hotMiddleware = {}
-    this.perfLoader = null
 
     // Initialize shared FS and Cache
     if (this.context.options.dev) {
@@ -37,8 +36,6 @@ export class WebpackBundler {
   async build() {
     const options = this.context.options
 
-    this.perfLoader = new PerfLoader(options)
-
     const compilersOptions = []
 
     // Client
@@ -104,7 +101,7 @@ export class WebpackBundler {
     // Warmup perfLoader before build
     if (options.build.parallel) {
       consola.info('Warming up worker pools')
-      this.perfLoader.warmupAll()
+      PerfLoader.warmupAll({ dev: options.dev })
       consola.success('Worker pools ready')
     }
 
diff --git a/packages/webpack/src/config/base.js b/packages/webpack/src/config/base.js
index 78e0f5b513..519f1c0425 100644
--- a/packages/webpack/src/config/base.js
+++ b/packages/webpack/src/config/base.js
@@ -10,6 +10,7 @@ import env from 'std-env'
 
 import { isUrl, urlJoin } from '@nuxt/common'
 
+import PerfLoader from './utils/perf-loader'
 import StyleLoader from './utils/style-loader'
 import WarnFixPlugin from './plugins/warnfix'
 
@@ -125,14 +126,13 @@ export default class WebpackBaseConfig {
   }
 
   rules() {
+    const perfLoader = new PerfLoader(this)
     const styleLoader = new StyleLoader(
       this.options,
       this.nuxt,
-      { isServer: this.isServer }
+      { isServer: this.isServer, perfLoader }
     )
 
-    const perfLoader = this.builder.perfLoader
-
     return [
       {
         test: /\.vue$/,
@@ -173,46 +173,46 @@ export default class WebpackBaseConfig {
 
           return !modulesToTranspile.some(module => module.test(file))
         },
-        use: perfLoader.pool('js', {
+        use: perfLoader.js().concat({
           loader: require.resolve('babel-loader'),
           options: this.getBabelOptions()
         })
       },
       {
         test: /\.css$/,
-        oneOf: perfLoader.poolOneOf('css', styleLoader.apply('css'))
+        oneOf: styleLoader.apply('css')
       },
       {
         test: /\.less$/,
-        oneOf: perfLoader.poolOneOf('css', styleLoader.apply('less', {
+        oneOf: styleLoader.apply('less', {
           loader: 'less-loader',
           options: this.loaders.less
-        }))
+        })
       },
       {
         test: /\.sass$/,
-        oneOf: perfLoader.poolOneOf('css', styleLoader.apply('sass', {
+        oneOf: styleLoader.apply('sass', {
           loader: 'sass-loader',
           options: this.loaders.sass
-        }))
+        })
       },
       {
         test: /\.scss$/,
-        oneOf: perfLoader.poolOneOf('css', styleLoader.apply('scss', {
+        oneOf: styleLoader.apply('scss', {
           loader: 'sass-loader',
           options: this.loaders.scss
-        }))
+        })
       },
       {
         test: /\.styl(us)?$/,
-        oneOf: perfLoader.poolOneOf('css', styleLoader.apply('stylus', {
+        oneOf: styleLoader.apply('stylus', {
           loader: 'stylus-loader',
           options: this.loaders.stylus
-        }))
+        })
       },
       {
         test: /\.(png|jpe?g|gif|svg|webp)$/,
-        use: perfLoader.pool('assets', {
+        use: perfLoader.asset().concat({
           loader: 'url-loader',
           options: Object.assign(
             this.loaders.imgUrl,
@@ -222,7 +222,7 @@ export default class WebpackBaseConfig {
       },
       {
         test: /\.(woff2?|eot|ttf|otf)(\?.*)?$/,
-        use: perfLoader.pool('assets', {
+        use: perfLoader.asset().concat({
           loader: 'url-loader',
           options: Object.assign(
             this.loaders.fontUrl,
@@ -232,7 +232,7 @@ export default class WebpackBaseConfig {
       },
       {
         test: /\.(webm|mp4|ogv)$/,
-        use: perfLoader.pool('assets', {
+        use: perfLoader.asset().concat({
           loader: 'file-loader',
           options: Object.assign(
             this.loaders.file,
diff --git a/packages/webpack/src/config/utils/perf-loader.js b/packages/webpack/src/config/utils/perf-loader.js
index 5414c84664..2d888d2db0 100644
--- a/packages/webpack/src/config/utils/perf-loader.js
+++ b/packages/webpack/src/config/utils/perf-loader.js
@@ -6,58 +6,58 @@ import { warmup } from 'thread-loader'
 // https://github.com/webpack-contrib/cache-loader
 
 export default class PerfLoader {
-  constructor(options) {
-    this.options = options
-    this.warmup = warmup
-    this.workerPools = {
-      js: {
-        name: 'js',
-        poolTimeout: this.options.dev ? Infinity : 2000
-      },
-      css: {
-        name: 'css',
-        poolTimeout: this.options.dev ? Infinity : 2000
+  constructor(config) {
+    this.name = config.name
+    this.options = config.options
+    this.workerPools = PerfLoader.defaultPools(this.options)
+    return new Proxy(this, {
+      get(target, name) {
+        return target[name] ? target[name] : target.use.bind(target, name)
       }
+    })
+  }
+
+  static defaultPools({ dev }) {
+    const poolTimeout = dev ? Infinity : 2000
+    return {
+      js: { name: 'js', poolTimeout },
+      css: { name: 'css', poolTimeout }
     }
   }
 
-  warmupAll() {
-    this.warmup(this.workerPools.js, [
+  static warmupAll(options) {
+    options = PerfLoader.defaultPools(options)
+    PerfLoader.warmup(options.js, [
       require.resolve('babel-loader'),
       require.resolve('@babel/preset-env')
     ])
-    this.warmup(this.workerPools.css, ['css-loader'])
+    PerfLoader.warmup(options.css, ['css-loader'])
   }
 
-  pool(poolName, _loaders) {
-    const loaders = [].concat(_loaders)
+  use(poolName) {
+    const loaders = []
+
+    if (this.options.build.cache) {
+      loaders.push({
+        loader: 'cache-loader',
+        options: {
+          cacheDirectory: path.resolve(`node_modules/.cache/cache-loader/${this.name}`)
+        }
+      })
+    }
 
     if (this.options.build.parallel) {
       const pool = this.workerPools[poolName]
-
       if (pool) {
-        loaders.unshift({
+        loaders.push({
           loader: 'thread-loader',
           options: pool
         })
       }
     }
 
-    if (this.options.build.cache) {
-      loaders.unshift({
-        loader: 'cache-loader',
-        options: {
-          cacheDirectory: path.resolve('node_modules/.cache/cache-loader')
-        }
-      })
-    }
-
     return loaders
   }
-
-  poolOneOf(poolName, oneOfRules) {
-    return oneOfRules.map(rule => Object.assign({}, rule, {
-      use: this.pool(poolName, rule.use)
-    }))
-  }
 }
+
+PerfLoader.warmup = warmup
diff --git a/packages/webpack/src/config/utils/style-loader.js b/packages/webpack/src/config/utils/style-loader.js
index cd94e29da1..daba33459b 100644
--- a/packages/webpack/src/config/utils/style-loader.js
+++ b/packages/webpack/src/config/utils/style-loader.js
@@ -6,8 +6,9 @@ import { wrapArray } from '@nuxt/common'
 import PostcssConfig from './postcss'
 
 export default class StyleLoader {
-  constructor(options, nuxt, { isServer }) {
+  constructor(options, nuxt, { isServer, perfLoader }) {
     this.isServer = isServer
+    this.perfLoader = perfLoader
     this.dev = options.dev
     this.srcDir = options.srcDir
     this.assetsDir = options.dir.assets
@@ -104,7 +105,7 @@ export default class StyleLoader {
       // This matches <style module>
       {
         resourceQuery: /module/,
-        use: [].concat(
+        use: this.perfLoader.css().concat(
           styleLoader,
           this.cssModules(cssModulesOptions),
           customLoaders
@@ -112,7 +113,7 @@ export default class StyleLoader {
       },
       // This matches plain <style> or <style scoped>
       {
-        use: [].concat(
+        use: this.perfLoader.css().concat(
           styleLoader,
           this.css(cssOptions),
           customLoaders
diff --git a/test/unit/wp.config.test.js b/test/unit/wp.config.test.js
index 28302c13d1..1641c03362 100644
--- a/test/unit/wp.config.test.js
+++ b/test/unit/wp.config.test.js
@@ -4,32 +4,33 @@ import PerfLoader from '../../packages/webpack/src/config/utils/perf-loader'
 
 describe('webpack configuration', () => {
   test('performance loader', () => {
-    const perfLoader = new PerfLoader({
-      dev: true,
-      build: {
-        parallel: true,
-        cache: true
-      }
-    })
     const js = { name: 'js', poolTimeout: Infinity }
     const css = { name: 'css', poolTimeout: Infinity }
-    expect(perfLoader.workerPools).toMatchObject({ js, css })
-
-    perfLoader.warmup = jest.fn()
-    perfLoader.warmupAll()
-    expect(perfLoader.warmup).toHaveBeenCalledTimes(2)
-    expect(perfLoader.warmup).toHaveBeenCalledWith(js, [
+    PerfLoader.warmup = jest.fn()
+    PerfLoader.warmupAll({ dev: true })
+    expect(PerfLoader.warmup).toHaveBeenCalledTimes(2)
+    expect(PerfLoader.warmup).toHaveBeenCalledWith(js, [
       require.resolve('babel-loader'),
       require.resolve('@babel/preset-env')
     ])
-    expect(perfLoader.warmup).toHaveBeenCalledWith(css, ['css-loader'])
+    expect(PerfLoader.warmup).toHaveBeenCalledWith(css, ['css-loader'])
 
-    const loaders = perfLoader.pool('js', { loader: 'test-perf-loader' })
-    const cacheDirectory = path.resolve('node_modules/.cache/cache-loader')
+    const perfLoader = new PerfLoader({
+      name: 'test-perf',
+      options: {
+        dev: true,
+        build: {
+          parallel: true,
+          cache: true
+        }
+      }
+    })
+    expect(perfLoader.workerPools).toMatchObject({ js, css })
+    const loaders = perfLoader.use('js')
+    const cacheDirectory = path.resolve('node_modules/.cache/cache-loader/test-perf')
     expect(loaders).toMatchObject([
       { loader: 'cache-loader', options: { cacheDirectory } },
-      { loader: 'thread-loader', options: js },
-      { loader: 'test-perf-loader' }
+      { loader: 'thread-loader', options: js }
     ])
   })
 })