diff --git a/src/litserve/server.py b/src/litserve/server.py
index 92ee4859..dc98974c 100644
--- a/src/litserve/server.py
+++ b/src/litserve/server.py
@@ -275,7 +275,8 @@ def __init__(
 
     def launch_inference_worker(self, num_uvicorn_servers: int):
         self.broker = Broker()
-        self.broker.start()
+        if self.use_zmq:
+            self.broker.start()
         manager = mp.Manager()
         self.workers_setup_status = manager.dict()
         self.request_queue = manager.Queue()