diff --git a/packages/vite/src/client.ts b/packages/vite/src/client.ts
index 44397003b7..12d746c021 100644
--- a/packages/vite/src/client.ts
+++ b/packages/vite/src/client.ts
@@ -171,18 +171,22 @@ export async function buildClient (ctx: ViteBuildContext) {
   }) as any
 
   if (clientConfig.server && clientConfig.server.hmr !== false) {
-    const hmrPortDefault = 24678 // Vite's default HMR port
-    const hmrPort = await getPort({
-      port: hmrPortDefault,
-      ports: Array.from({ length: 20 }, (_, i) => hmrPortDefault + 1 + i),
-    })
-    clientConfig.server = defu(clientConfig.server, <ServerOptions> {
-      https: ctx.nuxt.options.devServer.https,
+    const serverDefaults: Omit<ServerOptions, 'hmr'> & { hmr: Exclude<ServerOptions['hmr'], boolean> } = {
       hmr: {
         protocol: ctx.nuxt.options.devServer.https ? 'wss' : 'ws',
-        port: hmrPort,
       },
-    })
+    }
+    if (typeof clientConfig.server.hmr !== 'object' || !clientConfig.server.hmr.server) {
+      const hmrPortDefault = 24678 // Vite's default HMR port
+      serverDefaults.hmr!.port = await getPort({
+        port: hmrPortDefault,
+        ports: Array.from({ length: 20 }, (_, i) => hmrPortDefault + 1 + i),
+      })
+    }
+    if (ctx.nuxt.options.devServer.https) {
+      serverDefaults.https = ctx.nuxt.options.devServer.https === true ? {} : ctx.nuxt.options.devServer.https
+    }
+    clientConfig.server = defu(clientConfig.server, serverDefaults as ViteConfig['server'])
   }
 
   // Add analyze plugin if needed