diff --git a/backend/backend.proto b/backend/backend.proto
index 85e872607a3..90bcb2bc623 100644
--- a/backend/backend.proto
+++ b/backend/backend.proto
@@ -233,6 +233,8 @@ message ModelOptions {
 
   bool FlashAttention = 56;
   bool NoKVOffload = 57;
+
+  string ModelPath = 59;
 }
 
 message Result {
diff --git a/backend/python/diffusers/backend.py b/backend/python/diffusers/backend.py
index 087b449e4e3..db91789f71a 100755
--- a/backend/python/diffusers/backend.py
+++ b/backend/python/diffusers/backend.py
@@ -301,13 +301,11 @@ def LoadModel(self, request, context):
                 self.pipe.controlnet = self.controlnet
             else:
                 self.controlnet = None
-            # Assume directory from request.ModelFile.
-            # Only if request.LoraAdapter it's not an absolute path
-            if request.LoraAdapter and request.ModelFile != "" and not os.path.isabs(request.LoraAdapter) and request.LoraAdapter:
-                # get base path of modelFile
-                modelFileBase = os.path.dirname(request.ModelFile)
+
+            if request.LoraAdapter and not os.path.isabs(request.LoraAdapter):
                 # modify LoraAdapter to be relative to modelFileBase
-                request.LoraAdapter = os.path.join(modelFileBase, request.LoraAdapter)
+                request.LoraAdapter = os.path.join(request.ModelPath, request.LoraAdapter)
+
             device = "cpu" if not request.CUDA else "cuda"
             self.device = device
             if request.LoraAdapter:
diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go
index bd668ec25a2..5723e3e41db 100644
--- a/pkg/model/initializers.go
+++ b/pkg/model/initializers.go
@@ -425,6 +425,7 @@ func (ml *ModelLoader) grpcModel(backend string, autodetect bool, o *Options) fu
 		options := *o.gRPCOptions
 		options.Model = modelName
 		options.ModelFile = modelFile
+		options.ModelPath = ml.ModelPath
 
 		log.Debug().Msgf("GRPC: Loading model with options: %+v", options)