From 0e378d89493218d3f5116e967473cab058321005 Mon Sep 17 00:00:00 2001 From: Ean Garvey Date: Thu, 21 Nov 2024 17:55:11 -0600 Subject: [PATCH] (shortfin-sd) Interleave fibers by device. --- shortfin/python/shortfin_apps/sd/components/service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shortfin/python/shortfin_apps/sd/components/service.py b/shortfin/python/shortfin_apps/sd/components/service.py index 9b09632a6..814916bee 100644 --- a/shortfin/python/shortfin_apps/sd/components/service.py +++ b/shortfin/python/shortfin_apps/sd/components/service.py @@ -76,8 +76,10 @@ def __init__( self.workers = [] self.fibers = [] self.idle_fibers = set() - for idx, device in enumerate(self.sysman.ls.devices): - for i in range(self.workers_per_device): + # For each worker index we create one on each device, and add their fibers to the idle set. + # This roughly ensures that the first picked fibers are distributed across available devices. + for i in range(self.workers_per_device): + for idx, device in enumerate(self.sysman.ls.devices): worker = sysman.ls.create_worker(f"{name}-inference-{device.name}-{i}") self.workers.append(worker) for idx, device in enumerate(self.sysman.ls.devices):