From 54d784565c635785e8eca0a60e2a97874ff385af Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Thu, 3 Oct 2024 18:59:07 +0800 Subject: [PATCH 01/27] Fixed indexing bug on intervals --- .../heuristic_solver/task_allocator.py | 64 +++++++++++++------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 927c67f..6e529ba 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -448,37 +448,61 @@ def _create_assignments_matrix( return assignments_matrix + + def _find_indexes(self, arr: np.array) -> tuple[int, int] | None: """ - Find the start and end indexes from the last zero to the last number with no increase in a NumPy array. + Find the start and end indexes for a valid segment of resource availability. + This version avoids explicit loops and ensures the start index is correctly identified. """ - # if last element is zero return None - if arr[-1] == 0: - return None - - # Find the index of the last zero - zero_indexes = np.nonzero(arr == 0)[0] - if zero_indexes.size > 0: - start_index = zero_indexes[-1] + # If the input is a MaskedArray, handle it accordingly + if isinstance(arr, np.ma.MaskedArray): + arr_data = arr.data + mask = arr.mask + # Find valid (unmasked and positive) indices + valid_indices = np.where((~mask) & (arr_data >= 0))[0] else: + valid_indices = np.where(arr >= 0)[0] + + # If no valid indices are found, return None (no available resources) + if valid_indices.size == 0: return None - # Use np.diff to find where the array stops increasing - diffs = np.diff(arr[start_index:]) + # Identify if the start of the array is valid + start_index = 0 if arr[0] > 0 else valid_indices[0] - # Find where the difference is less than or equal to zero (non-increasing sequence) - non_increasing = np.where(diffs == 0)[0] + # Calculate differences between consecutive indices + diffs = np.diff(valid_indices) - if non_increasing.size > 0: - # The end index is the last non-increasing index + 1 to account for the difference in np.diff indexing - end_index = non_increasing[0] + start_index - else: - end_index = ( - arr.size - 1 - ) # If the array always increases, end at the last index + # Identify segment boundaries where there is a gap greater than 1 + gaps = diffs > 1 + segment_boundaries = np.where(gaps)[0] + + # Insert the start index explicitly to ensure it is considered + segment_starts = np.insert(segment_boundaries + 1, 0, 0) + segment_ends = np.append(segment_starts[1:], len(valid_indices)) + + # Always take the first segment (which starts at the earliest valid index) + start_pos = segment_starts[0] + end_pos = segment_ends[0] - 1 + + # Convert these segment positions to the actual start and end indices + start_index = valid_indices[start_pos] + end_index = valid_indices[end_pos] + + # Debugging statements + print(f"Valid indices: {valid_indices}") + print(f"Diffs: {diffs}") + print(f"Segment boundaries: {segment_boundaries}") + print(f"Segment starts: {segment_starts}") + print(f"Segment ends: {segment_ends}") + print(f"Selected start index: {start_index}") + print(f"Selected end index: {end_index}") return start_index, end_index + + def _linear_interpolate_nan(self, y: np.ndarray, x: np.ndarray) -> np.ndarray: """ Linearly interpolate NaN values in a 1D array. From ecc5ccf4c361786741e81580a244eda98948b620 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Fri, 4 Oct 2024 14:58:42 +0800 Subject: [PATCH 02/27] Task Allocator stable --- nb.ipynb | 274 ++++++++++++++++++ .../scheduler/heuristic_solver/main.py | 10 +- .../heuristic_solver/task_allocator.py | 32 +- .../heuristic_solver/window_manager.py | 21 +- 4 files changed, 319 insertions(+), 18 deletions(-) create mode 100644 nb.ipynb diff --git a/nb.ipynb b/nb.ipynb new file mode 100644 index 0000000..6f73569 --- /dev/null +++ b/nb.ipynb @@ -0,0 +1,274 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Valid indices: [0 1 2 3]\n", + "Diffs: [1 1 1]\n", + "Segment boundaries: []\n", + "Segment starts: [0]\n", + "Segment ends: [4]\n", + "Selected start index: 0\n", + "Selected end index: 3\n", + "Indexes for resource 2: (0, 3)\n", + "Resource intervals: [ 0. 10. 20. 30.]\n", + "Valid indices: [0 1 2 3]\n", + "Diffs: [1 1 1]\n", + "Segment boundaries: []\n", + "Segment starts: [0]\n", + "Segment ends: [4]\n", + "Selected start index: 0\n", + "Selected end index: 3\n", + "Indexes for resource 1: (0, 3)\n", + "Resource intervals: [ 0. 10. 20. 30.]\n", + "Allocated resources for task 1: {2: [(0, 10), (20, 30)], 1: [(0, 10), (20, 30)]}\n", + "PASS\n", + "Scheduled 1 of 1 tasks.\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
task_idassigned_resource_idstask_starttask_endresource_intervals
01[2, 1]030([(0, 10), (20, 30)], [(0, 10), (20, 30)])
\n", + "
" + ], + "text/plain": [ + " task_id assigned_resource_ids task_start task_end \\\n", + "0 1 [2, 1] 0 30 \n", + "\n", + " resource_intervals \n", + "0 ([(0, 10), (20, 30)], [(0, 10), (20, 30)]) " + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.scheduler.core import Scheduler\n", + "\n", + "machine = Resource(id=1, available_windows=[(0, 50)])\n", + "operator1 = Resource(id=2, available_windows=[(0, 10), (20, 30)])\n", + "operator2 = Resource(id=3, available_windows=[(0, 10), (40, 50)])\n", + "\n", + "operator_group = ResourceGroup(resources=[operator1, operator2])\n", + "\n", + "assignment = Assignment(resource_groups=[operator_group], resource_count=1)\n", + "\n", + "# add machine as a constraint\n", + "t1 = Task(\n", + " id=1, duration=20, assignments=[assignment], priority=1, constraints=[machine]\n", + ")\n", + "\n", + "\n", + "tasks = [t1]\n", + "resources = [operator1, operator2, machine]\n", + "\n", + "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", + "result.to_dataframe()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Valid indices: [0 1 2 3]\n", + "Diffs: [1 1 1]\n", + "Segment boundaries: []\n", + "Segment starts: [0]\n", + "Segment ends: [4]\n", + "Selected start index: 0\n", + "Selected end index: 3\n", + "Indexes for resource 1: (0, 3)\n", + "Resource intervals: [ 0. 10. 40. 45.]\n", + "Allocated resources for task 1: {1: [(0, 10), (40, 45)]}\n", + "PASS\n", + "Scheduled 1 of 1 tasks.\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'task_id': 1,\n", + " 'assigned_resource_ids': [1],\n", + " 'task_start': 0,\n", + " 'task_end': 45,\n", + " 'resource_intervals': dict_values([[(0, 10), (40, 45)]])}]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.scheduler.core import Scheduler\n", + "\n", + "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 50)])\n", + "\n", + "operator_group = ResourceGroup(resources=[operator1])\n", + "\n", + "assignment = Assignment(resource_groups=[operator_group], resource_count=2)\n", + "\n", + "# add machine as a constraint\n", + "t1 = Task(\n", + " id=1, duration=15, assignments=[assignment], priority=1\n", + ")\n", + "\n", + "tasks = [t1]\n", + "resources = [operator1]\n", + "\n", + "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", + "result.to_dict()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Valid indices: [0 1 2 3]\n", + "Diffs: [1 1 1]\n", + "Segment boundaries: []\n", + "Segment starts: [0]\n", + "Segment ends: [4]\n", + "Selected start index: 0\n", + "Selected end index: 3\n", + "Indexes for resource 1: (0, 3)\n", + "Resource intervals: [ 0. 10. 20. 25.]\n", + "Allocated resources for task 1: {1: [(0, 10), (20, 25)]}\n", + "PASS\n", + "Valid indices: [0 1]\n", + "Diffs: [1]\n", + "Segment boundaries: []\n", + "Segment starts: [0]\n", + "Segment ends: [2]\n", + "Selected start index: 0\n", + "Selected end index: 1\n", + "Indexes for resource 1: (0, 1)\n", + "Resource intervals: [25. 40.]\n", + "Allocated resources for task 2: {1: [(25, 40)]}\n", + "PASS\n", + "Scheduled 2 of 2 tasks.\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'task_id': 1,\n", + " 'assigned_resource_ids': [1],\n", + " 'task_start': 0,\n", + " 'task_end': 25,\n", + " 'resource_intervals': dict_values([[(0, 10), (20, 25)]])},\n", + " {'task_id': 2,\n", + " 'assigned_resource_ids': [1],\n", + " 'task_start': 25,\n", + " 'task_end': 40,\n", + " 'resource_intervals': dict_values([[(25, 40)]])}]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.scheduler.core import Scheduler\n", + "\n", + "machine = Resource(id=1, available_windows=[(0, 10), (20, 45)])\n", + "\n", + "# add machine as a constraint\n", + "t1 = Task(\n", + " id=1, duration=15, priority=1, constraints=[machine]\n", + ")\n", + "\n", + "# add machine as a constraint\n", + "t2 = Task(\n", + " id=2, duration=15, priority=2, constraints=[machine]\n", + ")\n", + "\n", + "tasks = [t1, t2]\n", + "resources = [machine]\n", + "\n", + "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", + "result.to_dict()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py index 4b5bb52..b356505 100644 --- a/src/factryengine/scheduler/heuristic_solver/main.py +++ b/src/factryengine/scheduler/heuristic_solver/main.py @@ -75,23 +75,27 @@ def solve(self) -> list[dict]: self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e)) continue + print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}") + + print('PASS') # update resource windows self.window_manager.update_resource_windows(allocated_resource_windows_dict) - # Append task values + task_values = { "task_id": task_id, "assigned_resource_ids": list(allocated_resource_windows_dict.keys()), "task_start": min( - start for start, _ in allocated_resource_windows_dict.values() + start for intervals in allocated_resource_windows_dict.values() for start, _ in intervals ), "task_end": max( - end for _, end in allocated_resource_windows_dict.values() + end for intervals in allocated_resource_windows_dict.values() for _, end in intervals ), "resource_intervals": allocated_resource_windows_dict.values(), } self.task_vars[task_id] = task_values + return list( self.task_vars.values() ) # Return values of the dictionary as a list diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 6e529ba..1e95c40 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -162,26 +162,34 @@ def _solve_task_end( def _get_resource_intervals( self, matrix: np.array, - ) -> dict[int, tuple[int, int]]: + ) -> dict[int, list[tuple[int, int]]]: """ - gets the resource intervals from the solution matrix. + Gets the resource intervals from the solution matrix by pairing the intervals directly, + taking consecutive pairs like (index0, index1), (index2, index3), until the end index. """ - end_index = matrix.resource_matrix.shape[0] - 1 resource_windows_dict = {} - # loop through resource ids and resource intervals - for resource_id, resource_intervals in zip( - matrix.resource_ids, matrix.resource_matrix.T - ): - # ensure only continuous intervals are selected + + # Loop through resource ids and resource intervals + for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): + # Ensure only continuous intervals are selected indexes = self._find_indexes(resource_intervals.data) + print(f"Indexes for resource {resource_id}: {indexes}") + print(f"Resource intervals: {matrix.intervals}") + if indexes is not None: start_index, end_index = indexes - resource_windows_dict[resource_id] = ( - ceil(round(matrix.intervals[start_index], 1)), - ceil(round(matrix.intervals[end_index], 1)), - ) + + # Create pairs of intervals, using every two consecutive values until the end_index + segment_intervals = [ + (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1))) + for i in range(start_index, end_index, 2) # Step by 2 to get consecutive pairs + ] + + resource_windows_dict[resource_id] = segment_intervals + return resource_windows_dict + def _mask_smallest_elements_except_top_k_per_row( self, array: np.ma.core.MaskedArray, k ) -> np.ma.core.MaskedArray: diff --git a/src/factryengine/scheduler/heuristic_solver/window_manager.py b/src/factryengine/scheduler/heuristic_solver/window_manager.py index 6cb8324..32cb63e 100644 --- a/src/factryengine/scheduler/heuristic_solver/window_manager.py +++ b/src/factryengine/scheduler/heuristic_solver/window_manager.py @@ -45,14 +45,29 @@ def update_resource_windows( self, allocated_resource_windows_dict: dict[int, list[tuple[int, int]]] ) -> None: """ - Removes the task interaval from the resource windows + Removes the allocated intervals from the resource windows. """ - for resource_id, trim_interval in allocated_resource_windows_dict.items(): + for resource_id, trim_intervals in allocated_resource_windows_dict.items(): + if not trim_intervals: + continue + + # Get the earliest start and latest end of the intervals + combined_start = trim_intervals[0][0] + combined_end = trim_intervals[-1][1] + + # Create a single trim interval + combined_trim_interval = (combined_start, combined_end) + + # Get the window to trim window = self.resource_windows_dict[resource_id] + + # Trim the window using the combined interval self.resource_windows_dict[resource_id] = self._trim_window( - window, trim_interval + window, combined_trim_interval ) + + def _create_resource_windows_dict(self) -> dict[int, np.ndarray]: """ Creates a dictionary mapping resource IDs to numpy arrays representing windows. From 0597b99fa02895efad4a21852b4aba6f70d67124 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Fri, 4 Oct 2024 19:32:35 +0800 Subject: [PATCH 03/27] Updated task allocator + added test cases --- .../heuristic_solver/task_allocator.py | 11 +---- tests/scheduler/test_task_allocator.py | 45 +++++++++++-------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 1e95c40..14f16c0 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -169,6 +169,8 @@ def _get_resource_intervals( """ resource_windows_dict = {} + print(f"Matrix: {matrix}") + # Loop through resource ids and resource intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): # Ensure only continuous intervals are selected @@ -498,15 +500,6 @@ def _find_indexes(self, arr: np.array) -> tuple[int, int] | None: start_index = valid_indices[start_pos] end_index = valid_indices[end_pos] - # Debugging statements - print(f"Valid indices: {valid_indices}") - print(f"Diffs: {diffs}") - print(f"Segment boundaries: {segment_boundaries}") - print(f"Segment starts: {segment_starts}") - print(f"Segment ends: {segment_ends}") - print(f"Selected start index: {start_index}") - print(f"Selected end index: {end_index}") - return start_index, end_index diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index 772b979..d40d44c 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -28,17 +28,32 @@ def test_solve_task_end(task_allocator): assert np.array_equal(result_y, np.array([5, 5])) -def test_get_resource_intervals(task_allocator): - solution_resource_ids = np.array([1, 2, 3]) - solution_intervals = np.array([0, 1, 2]) - resource_matrix = np.ma.array([[0, 0, 0], [1, 0, 0], [2, 1, 0]]) +def test_get_resource_intervals_continuous(task_allocator): + # Test case continuous values 1 task 2 resources + solution_resource_ids = np.array([1, 2]) + solution_intervals = np.array([0, 1, 2, 3]) + resource_matrix = np.ma.array([[0, 0], [1, 1]]) solution_matrix = Matrix( resource_ids=solution_resource_ids, intervals=solution_intervals, resource_matrix=resource_matrix, ) result = task_allocator._get_resource_intervals(solution_matrix) - expeceted = {1: (0, 2), 2: (1, 2)} + expeceted = {1: [(0, 1)], 2: [(0, 1)]} + assert result == expeceted + +def test_get_resource_intervals_windowed(task_allocator): + # Test case windowed values 1 task 1 resource + solution_resource_ids = np.array([1]) + solution_intervals = np.array([0, 1, 4, 5, 7, 8]) + resource_matrix = np.ma.array([[0], [1], [4], [5], [7], [8]]) + solution_matrix = Matrix( + resource_ids=solution_resource_ids, + intervals=solution_intervals, + resource_matrix=resource_matrix, + ) + result = task_allocator._get_resource_intervals(solution_matrix) + expeceted = {1: [(0, 1), (4, 5), (7, 8)]} assert result == expeceted @@ -201,20 +216,12 @@ def test_diff_and_zero_negatives(array, expected): @pytest.mark.parametrize( - "array , expected", + "array, expected", [ - ( - np.array([0, 1, 2, 3, 4]), - (0, 4), - ), - ( - np.array([0, 3, 3, 3, 3]), - (0, 1), - ), - ( - np.array([0, 0, 1, 2, 0]), - None, - ), + # Full valid sequence without gaps + (np.array([0, 1, 2, 3, 4]), (0, 4)), + # Sequence with repeated values, expecting first valid segment + (np.array([0, 3]), (0, 1)), # This one might need revisiting if logic changes ], ) def test_find_indexes(array, expected): @@ -222,4 +229,4 @@ def test_find_indexes(array, expected): result = task_allocator._find_indexes(array) print("result :", result) print("expected:", expected) - np.testing.assert_array_equal(result, expected) + assert result == expected From 582677abda9e43cf549c9ad3f34f184326b87292 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Fri, 4 Oct 2024 19:32:44 +0800 Subject: [PATCH 04/27] Removed initial prints --- .../scheduler/heuristic_solver/task_allocator.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 14f16c0..4737e02 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -169,15 +169,10 @@ def _get_resource_intervals( """ resource_windows_dict = {} - print(f"Matrix: {matrix}") - # Loop through resource ids and resource intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): # Ensure only continuous intervals are selected indexes = self._find_indexes(resource_intervals.data) - print(f"Indexes for resource {resource_id}: {indexes}") - print(f"Resource intervals: {matrix.intervals}") - if indexes is not None: start_index, end_index = indexes From 7821a0ebaa4a5d7af7d0050d4d1a238839e134a2 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 8 Oct 2024 21:49:55 +0800 Subject: [PATCH 05/27] Core and test changes --- src/factryengine/models/task.py | 16 +++++++-- .../scheduler/heuristic_solver/main.py | 5 +-- .../scheduler/task_batch_processor.py | 33 +++++++++++++++++++ tests/scheduler/test_matrix.py | 1 - 4 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 src/factryengine/scheduler/task_batch_processor.py diff --git a/src/factryengine/models/task.py b/src/factryengine/models/task.py index cf04f20..046f4f0 100644 --- a/src/factryengine/models/task.py +++ b/src/factryengine/models/task.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field, model_validator, validator +from pydantic import BaseModel, Field, model_validator, validator, PrivateAttr from .resource import Resource, ResourceGroup @@ -44,15 +44,16 @@ def get_unique_resources(self) -> set[Resource]: class Task(BaseModel): - id: int + id: int | str name: str = "" duration: int = Field(gt=0) priority: int = Field(gt=0) assignments: list[Assignment] = [] constraints: set[Resource] = set() - predecessor_ids: set[int] = set() + predecessor_ids: set[int] | set[str] = set() predecessor_delay: int = Field(0, gt=0) quantity: int = Field(None, gt=0) + _batch_id: int = PrivateAttr(None) def __hash__(self): return hash(self.id) @@ -85,3 +86,12 @@ def set_name(cls, v, values) -> str: def get_id(self) -> int: """returns the task id""" return self.id + + @property + def batch_id(self): + """returns the batch id of the task""" + return self._batch_id + + def set_batch_id(self, batch_id): + """sets the batch id of the task""" + self._batch_id = batch_id diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py index b356505..cdb3f74 100644 --- a/src/factryengine/scheduler/heuristic_solver/main.py +++ b/src/factryengine/scheduler/heuristic_solver/main.py @@ -74,10 +74,7 @@ def solve(self) -> list[dict]: except AllocationError as e: self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e)) continue - - print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}") - - print('PASS') + # update resource windows self.window_manager.update_resource_windows(allocated_resource_windows_dict) diff --git a/src/factryengine/scheduler/task_batch_processor.py b/src/factryengine/scheduler/task_batch_processor.py new file mode 100644 index 0000000..b4e4083 --- /dev/null +++ b/src/factryengine/scheduler/task_batch_processor.py @@ -0,0 +1,33 @@ +from ..models import Resource, Task + +class TaskSplitter: + """ + The TaskSplitter class is responsible for splitting tasks into batches. + """ + + def __init__(self, task: Task, batch_size: int): + self.task = task + self.batch_size = batch_size + + def split_into_batches(self) -> list[Task]: + """ + Splits a task into batches. + """ + num_batches, remaining = divmod(self.task.quantity, self.batch_size) + batches = [ + self._create_new_task(i + 1, self.batch_size) + for i in range(num_batches) + ] + + if remaining > 0: + batches.append(self._create_new_task(num_batches + 1, remaining)) + + return batches + + def _create_new_task(self, batch_id: int, quantity: int) -> Task: + """Creates a new task with the given batch_id and quantity.""" + new_task = self.task.model_copy(deep=True) + new_task.quantity = quantity + new_task.duration = (quantity / self.task.quantity) * self.task.duration + new_task.set_batch_id(batch_id) + return new_task diff --git a/tests/scheduler/test_matrix.py b/tests/scheduler/test_matrix.py index 07a72b4..4348485 100644 --- a/tests/scheduler/test_matrix.py +++ b/tests/scheduler/test_matrix.py @@ -81,7 +81,6 @@ def test_can_compare_update_mask_and_merge(matrix_data_dict): [True, True, True, False, False, False], ] ) - print(merged_matrix.resource_matrix.mask) assert np.array_equal(merged_matrix.resource_matrix.mask, expected_mask) From 1e398303e8a935120398631cafcc98bb530b62a8 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 9 Oct 2024 17:01:25 +0800 Subject: [PATCH 06/27] Fixed get_resource_interval_function --- src/factryengine/scheduler/scheduler_result.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/factryengine/scheduler/scheduler_result.py b/src/factryengine/scheduler/scheduler_result.py index fdf3737..19b19d4 100644 --- a/src/factryengine/scheduler/scheduler_result.py +++ b/src/factryengine/scheduler/scheduler_result.py @@ -106,14 +106,21 @@ def get_resource_intervals_df(self) -> pd.DataFrame: # Drop any rows with missing values cleaned_df = exploded_df.dropna() + exploded_intervals_df = cleaned_df.explode("resource_intervals") + exploded_intervals_df = exploded_intervals_df.reset_index(drop=True) + # Extract the start and end of the interval from the 'resource_intervals' column - cleaned_df["interval_start"] = cleaned_df.resource_intervals.apply( + exploded_intervals_df["interval_start"] = exploded_intervals_df.resource_intervals.apply( lambda x: x[0] ) - cleaned_df["interval_end"] = cleaned_df.resource_intervals.apply(lambda x: x[1]) + + print('PASS INTERVAL START') + exploded_intervals_df["interval_end"] = exploded_intervals_df.resource_intervals.apply(lambda x: x[1]) + + print('PASS INTERVAL END') # Rename the 'assigned_resource_ids' column to 'resource_id' - renamed_df = cleaned_df.rename(columns={"assigned_resource_ids": "resource_id"}) + renamed_df = exploded_intervals_df.rename(columns={"assigned_resource_ids": "resource_id"}) # Select only the columns we're interested in selected_columns_df = renamed_df[ From 7f4de82f2900346c4b2e2d183b30e43c4e865c28 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 14 Oct 2024 18:35:12 +0800 Subject: [PATCH 07/27] Bugfix non-owned interval --- .../heuristic_solver/task_allocator.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 4737e02..7b0e855 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -164,24 +164,37 @@ def _get_resource_intervals( matrix: np.array, ) -> dict[int, list[tuple[int, int]]]: """ - Gets the resource intervals from the solution matrix by pairing the intervals directly, - taking consecutive pairs like (index0, index1), (index2, index3), until the end index. + Gets the resource intervals from the solution matrix. + Always includes the first pair, and only subsequent pairs + if value_end > value_start for any given pair. """ resource_windows_dict = {} - # Loop through resource ids and resource intervals + # Loop through resource IDs and corresponding intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): - # Ensure only continuous intervals are selected indexes = self._find_indexes(resource_intervals.data) + if indexes is not None: start_index, end_index = indexes - # Create pairs of intervals, using every two consecutive values until the end_index - segment_intervals = [ - (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1))) - for i in range(start_index, end_index, 2) # Step by 2 to get consecutive pairs - ] + segment_intervals = [] + is_first_pair = True # Track if this is the first pair + + # Iterate through the intervals in pairs (i, i+1) + for i in range(start_index, end_index, 2): + # Extract values from the resource matrix + value_start = matrix.resource_matrix[i][0] + value_end = matrix.resource_matrix[i + 1][0] + + # Always add the first pair, or add if value_end > value_start + if is_first_pair or value_end > value_start: + interval_start = ceil(round(matrix.intervals[i], 1)) + interval_end = ceil(round(matrix.intervals[i + 1], 1)) + + segment_intervals.append((interval_start, interval_end)) + is_first_pair = False # Switch off the first-pair flag + # Store the segment intervals for the current resource resource_windows_dict[resource_id] = segment_intervals return resource_windows_dict From e6585b70c8a88579666379ac78aff8090feffbd0 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 14 Oct 2024 21:23:27 +0800 Subject: [PATCH 08/27] Ignored inpults folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6b987e0..d8d9708 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Prototyping Notebooks notebooks/ +# Test data +inputs/ + # vscode settings .vscode/ From 0373f941f63e60ec912e9f82f915e236bdd861d8 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 15 Oct 2024 22:59:03 +0800 Subject: [PATCH 09/27] Constraints + assignments stable --- .../scheduler/heuristic_solver/matrix.py | 7 +- .../heuristic_solver/task_allocator.py | 64 +++++++++---------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/matrix.py b/src/factryengine/scheduler/heuristic_solver/matrix.py index 98f8248..8180538 100644 --- a/src/factryengine/scheduler/heuristic_solver/matrix.py +++ b/src/factryengine/scheduler/heuristic_solver/matrix.py @@ -56,9 +56,12 @@ def trim_end(cls, original_matrix: "Matrix", trim_matrix: "Matrix") -> "Matrix": Trims a Matrix based on another """ new_intervals = original_matrix.intervals[: len(trim_matrix.intervals)] - # Check if intervals are the same - if not np.array_equal(new_intervals, trim_matrix.intervals): + # if not np.array_equal(new_intervals, trim_matrix.intervals): + # raise ValueError("All matrices must have the same intervals") + + # Used np.allclose to allow for small differences in the intervals + if not np.allclose(new_intervals, trim_matrix.intervals, atol=1e-8): raise ValueError("All matrices must have the same intervals") return cls( diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 7b0e855..2501a94 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -218,17 +218,19 @@ def _mask_smallest_elements_except_top_k_per_row( def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray: """ - Computes the cumulative sum of an array but resets the sum to zero whenever a - -1 is encountered. This is a helper method used in the creation of the resource - windows matrix. + Computes the cumulative sum but resets to 0 whenever a -1 is encountered. """ - reset_at = a == -1 - a[reset_at] = 0 - without_reset = a.cumsum() - overcount = np.maximum.accumulate(without_reset * reset_at) - return without_reset - overcount + reset_mask = (a == -1) + a[reset_mask] = 0 # Replace -1 with 0 for sum calculation + cumsum_result = np.cumsum(a) + cumsum_result[reset_mask] = 0 # Reset at gaps + + return cumsum_result def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray: + """ + Applies cumulative sum along the columns of a 2D array and resets at gaps (-1). + """ return np.apply_along_axis(self._cumsum_reset_at_minus_one, axis=0, arr=arr) def _replace_masked_values_with_nan( @@ -362,52 +364,50 @@ def _create_resource_group_matrix( def _create_constraints_matrix( self, - resource_constraints: set[Resource], + resource_constraints: list[Resource], resource_windows_matrix: Matrix, task_duration: int, ) -> Matrix: """ - Checks if the resource constraints are available and updates the resource windows matrix. + Creates a constraints matrix by accumulating availability across multiple windows, + following the structure of the resource group matrix logic. """ if not resource_constraints: return None - # get the constraint resource ids + # Extract the resource IDs from the constraints resource_ids = np.array([resource.id for resource in resource_constraints]) - # check if all resource constraints are available - if not np.all(np.isin(resource_ids, resource_windows_matrix.resource_ids)): - raise AllocationError("All resource constraints are not available") + # Find the intersection of the resource constraints and the windows matrix + available_resources = np.intersect1d(resource_ids, resource_windows_matrix.resource_ids) + if len(available_resources) == 0: + return None - # Find the indices of the available resources in the windows matrix + # Find the indices of the relevant resources in the windows matrix resource_indexes = np.where( - np.isin(resource_windows_matrix.resource_ids, resource_ids) + np.isin(resource_windows_matrix.resource_ids, available_resources) )[0] - # get the windows for the resource constraints - constraint_windows = resource_windows_matrix.resource_matrix[ - :, resource_indexes - ] + # Extract the relevant windows from the matrix + constraint_matrix = resource_windows_matrix.resource_matrix[:, resource_indexes] - # Compute the minimum along axis 1, mask values <= 0, and compute the cumulative sum - # devide by the number of resources to not increase the task completion time - min_values_matrix = ( - np.min(constraint_windows, axis=1, keepdims=True) - * np.ones_like(constraint_windows) - / len(resource_ids) - ) + # Accumulate availability across windows using cumulative sum, resetting at gaps (-1) + accumulated_availability = self._cumsum_reset_at_minus_one_2d(constraint_matrix) + + # Check if accumulated availability meets the task duration requirement + if np.sum(accumulated_availability) < task_duration: + raise AllocationError("No solution found: Task duration exceeds available windows.") - resource_matrix = np.ma.masked_less_equal( - x=min_values_matrix, - value=0, - ).cumsum(axis=0) + # Mask zero values to represent unavailable slots + accumulated_availability = np.ma.masked_less_equal(accumulated_availability, 0) return Matrix( resource_ids=resource_ids, intervals=resource_windows_matrix.intervals, - resource_matrix=resource_matrix, + resource_matrix=accumulated_availability, ) + def _apply_constraint_to_resource_windows_matrix( self, constraint_matrix: Matrix, resource_windows_matrix: Matrix ) -> None: From 309c4b60b27abf7aec4a7aaf8c73fd06fa12d3b2 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 16 Oct 2024 14:36:26 +0800 Subject: [PATCH 10/27] Updated cumsum_reset and pytest --- .../heuristic_solver/task_allocator.py | 18 +++++++++++++----- tests/scheduler/test_task_allocator.py | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 2501a94..55a808f 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -220,12 +220,20 @@ def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray: """ Computes the cumulative sum but resets to 0 whenever a -1 is encountered. """ - reset_mask = (a == -1) - a[reset_mask] = 0 # Replace -1 with 0 for sum calculation - cumsum_result = np.cumsum(a) - cumsum_result[reset_mask] = 0 # Reset at gaps + a = a.copy() # Avoid in-place modification issues + result = np.zeros_like(a) # Initialize result array + + cumulative_sum = 0 # Track cumulative sum + for i in range(len(a)): + if a[i] == -1: + cumulative_sum = 0 # Reset cumulative sum + else: + cumulative_sum += a[i] + result[i] = cumulative_sum # Store result + + return result + - return cumsum_result def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray: """ diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index d40d44c..0a7a71f 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -133,7 +133,8 @@ def test_mask_smallest_elements_except_top_k_per_row( ) def test_cumsum_reset_at_minus_one(task_allocator, array, expected): result = task_allocator._cumsum_reset_at_minus_one(array) - assert np.array_equal(result, expected) + print(f"Input: {array}, Result: {result}, Expected: {expected}") + np.testing.assert_array_equal(result, expected) @pytest.mark.parametrize( From 0a313908ecd17dbf5c1e7166e0615910b4d47d11 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 16 Oct 2024 15:48:30 +0800 Subject: [PATCH 11/27] Reverted cumsum reset index function --- .../heuristic_solver/task_allocator.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 55a808f..2501a94 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -220,20 +220,12 @@ def _cumsum_reset_at_minus_one(self, a: np.ndarray) -> np.ndarray: """ Computes the cumulative sum but resets to 0 whenever a -1 is encountered. """ - a = a.copy() # Avoid in-place modification issues - result = np.zeros_like(a) # Initialize result array - - cumulative_sum = 0 # Track cumulative sum - for i in range(len(a)): - if a[i] == -1: - cumulative_sum = 0 # Reset cumulative sum - else: - cumulative_sum += a[i] - result[i] = cumulative_sum # Store result - - return result - + reset_mask = (a == -1) + a[reset_mask] = 0 # Replace -1 with 0 for sum calculation + cumsum_result = np.cumsum(a) + cumsum_result[reset_mask] = 0 # Reset at gaps + return cumsum_result def _cumsum_reset_at_minus_one_2d(self, arr: np.ndarray) -> np.ndarray: """ From 9ab20d1abdf7a9db9b2a5fea938e2b77d92273c7 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 16 Oct 2024 21:42:20 +0800 Subject: [PATCH 12/27] Current Updates --- nb.ipynb | 183 +++------- .../scheduler/heuristic_solver/main.py | 2 +- .../heuristic_solver/task_allocator.py | 65 ++-- stresstest.ipynb | 327 ++++++++++++++++++ stresstest.py | 234 +++++++++++++ 5 files changed, 645 insertions(+), 166 deletions(-) create mode 100644 stresstest.ipynb create mode 100644 stresstest.py diff --git a/nb.ipynb b/nb.ipynb index 6f73569..04e8393 100644 --- a/nb.ipynb +++ b/nb.ipynb @@ -2,33 +2,13 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Valid indices: [0 1 2 3]\n", - "Diffs: [1 1 1]\n", - "Segment boundaries: []\n", - "Segment starts: [0]\n", - "Segment ends: [4]\n", - "Selected start index: 0\n", - "Selected end index: 3\n", - "Indexes for resource 2: (0, 3)\n", - "Resource intervals: [ 0. 10. 20. 30.]\n", - "Valid indices: [0 1 2 3]\n", - "Diffs: [1 1 1]\n", - "Segment boundaries: []\n", - "Segment starts: [0]\n", - "Segment ends: [4]\n", - "Selected start index: 0\n", - "Selected end index: 3\n", - "Indexes for resource 1: (0, 3)\n", - "Resource intervals: [ 0. 10. 20. 30.]\n", - "Allocated resources for task 1: {2: [(0, 10), (20, 30)], 1: [(0, 10), (20, 30)]}\n", - "PASS\n", "Scheduled 1 of 1 tasks.\n" ] }, @@ -64,10 +44,10 @@ " \n", " 0\n", " 1\n", - " [2, 1]\n", + " [3, 1]\n", " 0\n", - " 30\n", - " ([(0, 10), (20, 30)], [(0, 10), (20, 30)])\n", + " 50\n", + " ([(0, 20), (30, 50)], [(0, 20), (30, 50)])\n", " \n", " \n", "\n", @@ -75,13 +55,13 @@ ], "text/plain": [ " task_id assigned_resource_ids task_start task_end \\\n", - "0 1 [2, 1] 0 30 \n", + "0 1 [3, 1] 0 50 \n", "\n", " resource_intervals \n", - "0 ([(0, 10), (20, 30)], [(0, 10), (20, 30)]) " + "0 ([(0, 20), (30, 50)], [(0, 20), (30, 50)]) " ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -92,8 +72,8 @@ "from src.factryengine.scheduler.core import Scheduler\n", "\n", "machine = Resource(id=1, available_windows=[(0, 50)])\n", - "operator1 = Resource(id=2, available_windows=[(0, 10), (20, 30)])\n", - "operator2 = Resource(id=3, available_windows=[(0, 10), (40, 50)])\n", + "operator1 = Resource(id=2, available_windows=[(0, 20), (30, 50)])\n", + "operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n", "\n", "operator_group = ResourceGroup(resources=[operator1, operator2])\n", "\n", @@ -101,10 +81,15 @@ "\n", "# add machine as a constraint\n", "t1 = Task(\n", - " id=1, duration=20, assignments=[assignment], priority=1, constraints=[machine]\n", + " id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n", ")\n", "\n", "\n", + "# # add machine as a constraint\n", + "# t2 = Task(\n", + "# id=2, duration=10, assignments=[assignment], priority=1, constraints=[machine]\n", + "# )\n", + "\n", "tasks = [t1]\n", "resources = [operator1, operator2, machine]\n", "\n", @@ -114,139 +99,53 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Valid indices: [0 1 2 3]\n", - "Diffs: [1 1 1]\n", - "Segment boundaries: []\n", - "Segment starts: [0]\n", - "Segment ends: [4]\n", - "Selected start index: 0\n", - "Selected end index: 3\n", - "Indexes for resource 1: (0, 3)\n", - "Resource intervals: [ 0. 10. 40. 45.]\n", - "Allocated resources for task 1: {1: [(0, 10), (40, 45)]}\n", - "PASS\n", - "Scheduled 1 of 1 tasks.\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'task_id': 1,\n", - " 'assigned_resource_ids': [1],\n", - " 'task_start': 0,\n", - " 'task_end': 45,\n", - " 'resource_intervals': dict_values([[(0, 10), (40, 45)]])}]" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.models import Resource, Task\n", "from src.factryengine.scheduler.core import Scheduler\n", "\n", - "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 50)])\n", - "\n", - "operator_group = ResourceGroup(resources=[operator1])\n", - "\n", - "assignment = Assignment(resource_groups=[operator_group], resource_count=2)\n", + "# Define a resource with multiple fragmented windows\n", + "machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n", + "machine2 = Resource(id=2, available_windows=[(0, 20), (60, 80)])\n", "\n", - "# add machine as a constraint\n", - "t1 = Task(\n", - " id=1, duration=15, assignments=[assignment], priority=1\n", - ")\n", + "# Create tasks with constraints\n", + "t1 = Task(id=1, duration=30, priority=1, constraints=[machine])\n", + "t2 = Task(id=2, duration=20, priority=2, constraints=[machine, machine2])\n", "\n", - "tasks = [t1]\n", - "resources = [operator1]\n", + "tasks = [t1, t2]\n", + "resources = [machine, machine2]\n", "\n", + "# Schedule the tasks\n", "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", - "result.to_dict()\n" + "result.to_dataframe()\n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Valid indices: [0 1 2 3]\n", - "Diffs: [1 1 1]\n", - "Segment boundaries: []\n", - "Segment starts: [0]\n", - "Segment ends: [4]\n", - "Selected start index: 0\n", - "Selected end index: 3\n", - "Indexes for resource 1: (0, 3)\n", - "Resource intervals: [ 0. 10. 20. 25.]\n", - "Allocated resources for task 1: {1: [(0, 10), (20, 25)]}\n", - "PASS\n", - "Valid indices: [0 1]\n", - "Diffs: [1]\n", - "Segment boundaries: []\n", - "Segment starts: [0]\n", - "Segment ends: [2]\n", - "Selected start index: 0\n", - "Selected end index: 1\n", - "Indexes for resource 1: (0, 1)\n", - "Resource intervals: [25. 40.]\n", - "Allocated resources for task 2: {1: [(25, 40)]}\n", - "PASS\n", - "Scheduled 2 of 2 tasks.\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'task_id': 1,\n", - " 'assigned_resource_ids': [1],\n", - " 'task_start': 0,\n", - " 'task_end': 25,\n", - " 'resource_intervals': dict_values([[(0, 10), (20, 25)]])},\n", - " {'task_id': 2,\n", - " 'assigned_resource_ids': [1],\n", - " 'task_start': 25,\n", - " 'task_end': 40,\n", - " 'resource_intervals': dict_values([[(25, 40)]])}]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n", "from src.factryengine.scheduler.core import Scheduler\n", "\n", - "machine = Resource(id=1, available_windows=[(0, 10), (20, 45)])\n", + "operator1 = Resource(id=1, available_windows=[(0, 20), (40, 60)])\n", + "operator2 = Resource(id=2, available_windows=[(0, 20), (40, 60)])\n", + "operator3 = Resource(id=3, available_windows=[(0, 30), (50, 60), (80, 150)])\n", "\n", - "# add machine as a constraint\n", - "t1 = Task(\n", - " id=1, duration=15, priority=1, constraints=[machine]\n", - ")\n", + "rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n", "\n", - "# add machine as a constraint\n", - "t2 = Task(\n", - " id=2, duration=15, priority=2, constraints=[machine]\n", - ")\n", + "assignment = Assignment(resource_groups=[rg1], resource_count=1)\n", "\n", - "tasks = [t1, t2]\n", - "resources = [machine]\n", + "t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n", + "t2 = Task(id=2, duration=20, assignments=[assignment], priority=2)\n", "\n", + "tasks = [t1, t2]\n", + "resources = [operator1, operator2, operator3]\n", "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", - "result.to_dict()\n" + "result.to_dict()" ] } ], diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py index cdb3f74..5b51052 100644 --- a/src/factryengine/scheduler/heuristic_solver/main.py +++ b/src/factryengine/scheduler/heuristic_solver/main.py @@ -74,7 +74,7 @@ def solve(self) -> list[dict]: except AllocationError as e: self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e)) continue - + # update resource windows self.window_manager.update_resource_windows(allocated_resource_windows_dict) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 2501a94..7ad2584 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -55,6 +55,7 @@ def allocate_task( # process solution to find allocated resource windows allocated_windows = self._get_resource_intervals( matrix=solution_matrix, + resource_windows_dict=resource_windows_dict ) # add constraints to allocated windows @@ -65,6 +66,7 @@ def allocate_task( allocated_windows.update( self._get_resource_intervals( matrix=constraints_matrix_trimmed, + resource_windows_dict=resource_windows_dict ) ) @@ -160,44 +162,61 @@ def _solve_task_end( return col0_value, other_columns_values def _get_resource_intervals( - self, - matrix: np.array, + self, matrix: Matrix, resource_windows_dict: dict[int, np.ndarray] ) -> dict[int, list[tuple[int, int]]]: """ - Gets the resource intervals from the solution matrix. - Always includes the first pair, and only subsequent pairs - if value_end > value_start for any given pair. + Extracts the resource intervals from the solution matrix by matching them + with the updated windows provided in `resource_windows_dict`. """ - resource_windows_dict = {} + resource_windows_output = {} - # Loop through resource IDs and corresponding intervals + # Iterate over each resource ID and its intervals in the solution matrix for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): indexes = self._find_indexes(resource_intervals.data) if indexes is not None: start_index, end_index = indexes - segment_intervals = [] - is_first_pair = True # Track if this is the first pair + # Extract allocated intervals from the matrix + allocated_intervals = [ + (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1))) + for i in range(start_index, end_index, 2) + ] + + # Get the resource’s available windows from the resource_windows_dict + updated_windows = resource_windows_dict.get(resource_id, []) + + # Match allocated intervals with the available windows + matched_intervals = self._match_intervals_with_windows( + allocated_intervals, updated_windows + ) - # Iterate through the intervals in pairs (i, i+1) - for i in range(start_index, end_index, 2): - # Extract values from the resource matrix - value_start = matrix.resource_matrix[i][0] - value_end = matrix.resource_matrix[i + 1][0] + # Store the matched intervals in the output dictionary + resource_windows_output[resource_id] = matched_intervals - # Always add the first pair, or add if value_end > value_start - if is_first_pair or value_end > value_start: - interval_start = ceil(round(matrix.intervals[i], 1)) - interval_end = ceil(round(matrix.intervals[i + 1], 1)) + return resource_windows_output + + def _match_intervals_with_windows( + self, allocated_intervals: list[tuple[int, int]], windows: np.ndarray + ) -> list[tuple[int, int]]: + """ + Matches the allocated intervals with the resource's available windows. + """ + matched_intervals = [] - segment_intervals.append((interval_start, interval_end)) - is_first_pair = False # Switch off the first-pair flag + # Iterate over allocated intervals and compare with the available windows + for allocated_start, allocated_end in allocated_intervals: + for window in windows: + window_start, window_end = window["start"], window["end"] - # Store the segment intervals for the current resource - resource_windows_dict[resource_id] = segment_intervals + # Check if there is an overlap between the allocated interval and the window + if allocated_end > window_start and allocated_start < window_end: + # Calculate the overlapping interval + matched_start = max(allocated_start, window_start) + matched_end = min(allocated_end, window_end) + matched_intervals.append((matched_start, matched_end)) - return resource_windows_dict + return matched_intervals def _mask_smallest_elements_except_top_k_per_row( diff --git a/stresstest.ipynb b/stresstest.ipynb new file mode 100644 index 0000000..e0f1769 --- /dev/null +++ b/stresstest.ipynb @@ -0,0 +1,327 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import json\n", + "import pytz\n", + "from datetime import datetime, timezone, timedelta\n", + "import time\n", + "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", + "from src.factryengine.scheduler.core import Scheduler\n", + "from src.factryengine.scheduler.task_batch_processor import TaskSplitter\n", + "\n", + "\n", + "class ProdScheduler: \n", + " def __init__(self, \n", + " resource_dir: str, \n", + " resource_group_dir: str, \n", + " task_dir: str\n", + " ) -> None:\n", + " # Scheduler Attributes\n", + " self.cph_timezone = pytz.timezone('Europe/Copenhagen')\n", + " # self.today = datetime.now(timezone.utc).replace(\n", + " # hour=0, minute=0, second=0, microsecond=0)\n", + " self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)\n", + " self.today_str = str(self.today)[:19]\n", + "\n", + " # Component Attributes\n", + " self.dict_resource = {}\n", + " self.dict_resourcegroups = {}\n", + " self.tasks_list = []\n", + " self.task_dict = {}\n", + " self.pred_dict = {}\n", + " self.flow_map = {}\n", + " self.pred_exploded = {}\n", + "\n", + " # Inputs \n", + " with open(resource_dir, 'r') as file:\n", + " self.r_data = json.load(file)\n", + "\n", + " with open(task_dir, 'r') as file:\n", + " self.t_data = json.load(file)\n", + "\n", + " with open(resource_group_dir, 'r') as file:\n", + " self.rg_data = json.load(file)\n", + " \n", + " def convert_to_minutes(self, datetime_str, start_time_obj):\n", + " datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')\n", + " diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60\n", + " return int(diff_minutes)\n", + "\n", + " def adjust_capacity(self, start, end, capacity):\n", + " return (end - start) * capacity + start\n", + "\n", + " def organize_predecessors(self, task: Task):\n", + " try:\n", + " list_predecessors = self.pred_dict[task.id]\n", + " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n", + " if task.batch_id: # Check if task is microbatched\n", + " # Look for each predecessors that exist in the flow map\n", + " for predecessor in list_predecessors:\n", + " pred_batch_id = f'{predecessor}-{task.batch_id}'\n", + " if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow\n", + " # Check if pred-parent connection is correct\n", + " if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:\n", + " self.pred_dict[task.id] = [pred_batch_id]\n", + "\n", + " elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:\n", + " parent_predecessor = []\n", + " for pred in self.pred_dict[task.id]:\n", + " parent_predecessor.extend(self.pred_dict[pred])\n", + " self.pred_dict[task.id] = parent_predecessor\n", + " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n", + "\n", + " # Remove task batch id\n", + " task.set_batch_id(None)\n", + "\n", + " # Check for predecessors to be exploded\n", + " for predecessor in list_predecessors:\n", + " if predecessor in self.pred_exploded:\n", + " self.pred_dict[task.id].remove(\n", + " predecessor) # Remove original value\n", + " self.pred_dict[task.id].extend(\n", + " self.pred_exploded[predecessor]) # Add exploded batches\n", + " except Exception as e:\n", + " return\n", + " \n", + " def set_predecessors(self, task: Task):\n", + " if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors\n", + " return\n", + "\n", + " for pred_id in self.pred_dict[task.id]:\n", + " if pred_id in self.task_dict: # ensure predecessor exists in task_dict\n", + " pred_task = self.task_dict[pred_id]\n", + "\n", + " # Avoid adding a predecessor multiple times\n", + " if pred_task not in task.predecessors:\n", + " # set predecessors for the predecessor first\n", + " self.set_predecessors(pred_task)\n", + " task.predecessors.append(pred_task)\n", + " \n", + " def create_resource_object(self, resource_list):\n", + " # Generate slot based on schedule selected in NocoDB\n", + " for row in resource_list:\n", + " periods_list = []\n", + " for sched in row['availability']:\n", + " if not sched['is_absent']:\n", + " start = self.convert_to_minutes(\n", + " sched['start_datetime'], self.today)\n", + " end = self.convert_to_minutes(\n", + " sched['end_datetime'], self.today)\n", + " # ========= Uncomment to use capacity\n", + " capacity = sched['capacity_percent']\n", + " # capacity = None\n", + " periods_list.append((int(start), int(self.adjust_capacity(\n", + " start, end, capacity)) if capacity else int(end)))\n", + "\n", + " resource_id = int(row['resource_id'])\n", + " self.dict_resource[resource_id] = Resource(\n", + " id=resource_id, available_windows=periods_list)\n", + "\n", + " def create_resource_groups(self, resource_group_list):\n", + " # Generate Resource Groups\n", + " for x in resource_group_list:\n", + " resource_list = []\n", + " resources = x['resource_id']\n", + " for r in resources:\n", + " if r in self.dict_resource:\n", + " resource_list.append(self.dict_resource[r])\n", + " \n", + " if resource_list:\n", + " self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(\n", + " id=x['resource_group_id'], resources=resource_list)\n", + "\n", + " def create_batch(self, task: Task, batch_size: int):\n", + " batches = TaskSplitter(task, batch_size).split_into_batches()\n", + " counter = 1\n", + " for batch in batches:\n", + " batch.id = f\"{task.id}-{counter}\"\n", + " counter += 1\n", + "\n", + " return batches\n", + "\n", + " def create_task_object(self, task_list):\n", + " for i in task_list:\n", + " rg_list = []\n", + " task_id = i['taskno']\n", + " duration = int(i['duration'])\n", + " priority = int(i['priority'])\n", + " quantity = int(i['quantity'])\n", + " # micro_batch_size = int(\n", + " # i['micro_batch_size']) if i['micro_batch_size'] else None\n", + " micro_batch_size = None\n", + " resource_group_id = i['resource_group_id']\n", + " rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]\n", + " predecessors = i['predecessors']\n", + " parent_collection = i['parent_item_collection_id'] if micro_batch_size else None\n", + " predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None\n", + "\n", + " assignments = []\n", + " # Create assignments \n", + " for x in resource_group_id: \n", + " if x in self.dict_resourcegroups:\n", + " assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))\n", + " \n", + " # Temporarily add into component dicts\n", + " temp_task = Task(id=task_id,\n", + " duration=duration,\n", + " priority=priority,\n", + " assignments= assignments,\n", + " quantity=quantity)\n", + " \n", + " # Check for micro-batches\n", + " if not micro_batch_size:\n", + " self.task_dict[task_id] = temp_task # Add task to dictionary\n", + "\n", + " # Add predecessor to dictionary\n", + " self.pred_dict[task_id] = predecessors\n", + " else:\n", + " self.pred_dict[task_id] = predecessors\n", + " batches = self.create_batch(temp_task, micro_batch_size)\n", + " self.task_dict.update({task.id: task for task in batches})\n", + "\n", + " # Temporarily copy the original predecessors for the new batches\n", + " self.pred_dict.update(\n", + " {task.id: predecessors for task in batches})\n", + " self.flow_map.update({task.id: {\n", + " \"parent\": parent_collection,\n", + " \"predecessor\": predecessor_collection} for task in batches})\n", + " self.pred_exploded[task_id] = [task.id for task in batches]\n", + "\n", + "\n", + " # Organize predecessors for batches\n", + " for task in self.task_dict.values():\n", + " self.organize_predecessors(task)\n", + "\n", + " # Add predecessors\n", + " for task in self.task_dict.values():\n", + " self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary\n", + "\n", + " # Build final task list\n", + " self.tasks_list = [value for key,\n", + " value in sorted(self.task_dict.items())]\n", + " \n", + " # Convert periods to time\n", + " def int_to_datetime(self, num, start_time):\n", + " try:\n", + " # Parse the start time string into a datetime object\n", + " start_datetime = datetime.strptime(start_time, \"%Y-%m-%d %H:%M:%S\")\n", + " \n", + " # Add the number of minutes to the start datetime\n", + " delta = timedelta(minutes=num)\n", + " result_datetime = start_datetime + delta\n", + " return result_datetime\n", + " \n", + " except Exception as e: \n", + " print(num)\n", + " \n", + " def run_scheduler(self):\n", + " self.create_resource_object(self.r_data)\n", + " print(\"Resource Objects Created.\")\n", + "\n", + " self.create_resource_groups(self.rg_data)\n", + " print(\"Resource Groups Created.\")\n", + "\n", + " self.create_task_object(self.t_data)\n", + " print(\"Task Objects Created.\")\n", + " print(f\"Original Task Length: {len(self.t_data)} | Post-Batched Length: {len(self.task_dict.values())}\")\n", + "\n", + "\n", + " self.solution = Scheduler(self.tasks_list, list(self.dict_resource.values())).schedule()\n", + " print(\"Solution Created.\")\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scheduler = ProdScheduler(\n", + " resource_dir = 'inputs/actual_data/resource.json',\n", + " resource_group_dir = 'inputs/actual_data/resourcegroups.json',\n", + " task_dir = 'inputs/actual_data/tasks_all.json'\n", + ")\n", + "\n", + "scheduler.run_scheduler()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "result = scheduler.solution.to_dataframe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result['start_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_start'], scheduler.today_str), axis=1)\n", + "result['end_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_end'], scheduler.today_str), axis=1)\n", + "result.sort_values(by='task_end', ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result[result['task_id'].str.startswith('WO137709')].sort_values(by='task_start')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scheduler.task_dict['WO135483-40']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "intervals = scheduler.solution.get_resource_intervals_df()\n", + "intervals['start_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_start'], scheduler.today_str), axis=1)\n", + "intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n", + "intervals.sort_values(by='interval_end', ascending=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/stresstest.py b/stresstest.py new file mode 100644 index 0000000..b1610ed --- /dev/null +++ b/stresstest.py @@ -0,0 +1,234 @@ +import pandas as pd +import json +import pytz +from datetime import datetime, timezone, timedelta +import time +from src.factryengine.models import Resource, Task, Assignment, ResourceGroup +from src.factryengine.scheduler.core import Scheduler +from src.factryengine.scheduler.task_batch_processor import TaskSplitter + + +class ProdScheduler: + def __init__(self) -> None: + # Scheduler Attributes + self.cph_timezone = pytz.timezone('Europe/Copenhagen') + # self.today = datetime.now(timezone.utc).replace( + # hour=0, minute=0, second=0, microsecond=0) + self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc) + self.today_str = str(self.today)[:19] + + # Component Attributes + self.dict_resource = {} + self.dict_resourcegroups = {} + self.tasks_list = [] + self.task_dict = {} + self.pred_dict = {} + self.flow_map = {} + self.pred_exploded = {} + + # Misc Attributes + self.start_time = time.time() + + def convert_to_minutes(self, datetime_str, start_time_obj): + datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z') + diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60 + return int(diff_minutes) + + def adjust_capacity(self, start, end, capacity): + return (end - start) * capacity + start + + def organize_predecessors(self, task: Task): + try: + list_predecessors = self.pred_dict[task.id] + # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW + if task.batch_id: # Check if task is microbatched + # Look for each predecessors that exist in the flow map + for predecessor in list_predecessors: + pred_batch_id = f'{predecessor}-{task.batch_id}' + if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow + # Check if pred-parent connection is correct + if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']: + self.pred_dict[task.id] = [pred_batch_id] + + elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict: + parent_predecessor = [] + for pred in self.pred_dict[task.id]: + parent_predecessor.extend(self.pred_dict[pred]) + self.pred_dict[task.id] = parent_predecessor + # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW + + # Remove task batch id + task.set_batch_id(None) + + # Check for predecessors to be exploded + for predecessor in list_predecessors: + if predecessor in self.pred_exploded: + self.pred_dict[task.id].remove( + predecessor) # Remove original value + self.pred_dict[task.id].extend( + self.pred_exploded[predecessor]) # Add exploded batches + + except Exception as e: + return + + def set_predecessors(self, task: Task): + if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors + return + + for pred_id in self.pred_dict[task.id]: + if pred_id in self.task_dict: # ensure predecessor exists in task_dict + pred_task = self.task_dict[pred_id] + + # Avoid adding a predecessor multiple times + if pred_task not in task.predecessors: + # set predecessors for the predecessor first + self.set_predecessors(pred_task) + task.predecessors.append(pred_task) + + def create_resource_object(self, resource_list): + # Generate slot based on schedule selected in NocoDB + for row in resource_list: + periods_list = [] + for sched in row['availability']: + if not sched['is_absent']: + start = self.convert_to_minutes( + sched['start_datetime'], self.today) + end = self.convert_to_minutes( + sched['end_datetime'], self.today) + # ========= Uncomment to use capacity + capacity = sched['capacity_percent'] + # capacity = None + periods_list.append((int(start), int(self.adjust_capacity( + start, end, capacity)) if capacity else int(end))) + + resource_id = int(row['resource_id']) + self.dict_resource[resource_id] = Resource( + id=resource_id, available_windows=periods_list) + + def create_resource_groups(self, resource_group_list): + # Generate Resource Groups + for x in resource_group_list: + resource_list = [] + resources = x['resource_id'] + for r in resources: + if r in self.dict_resource: + resource_list.append(self.dict_resource[r]) + + if resource_list: + self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup( + id=x['resource_group_id'], resources=resource_list) + + def create_batch(self, task: Task, batch_size: int): + batches = TaskSplitter(task, batch_size).split_into_batches() + counter = 1 + for batch in batches: + batch.id = f"{task.id}-{counter}" + counter += 1 + + return batches + + def create_task_object(self, task_list): + for i in task_list: + rg_list = [] + task_id = i['taskno'] + duration = int(i['duration']) + priority = int(i['priority']) + quantity = int(i['quantity']) + # micro_batch_size = int( + # i['micro_batch_size']) if i['micro_batch_size'] else None + micro_batch_size = None + resource_group_id = i['resource_group_id'] + rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups] + predecessors = i['predecessors'] + parent_collection = i['parent_item_collection_id'] if micro_batch_size else None + predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None + + assignments = [] + # Create assignments + for x in resource_group_id: + if x in self.dict_resourcegroups: + assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1)) + + # Temporarily add into component dicts + temp_task = Task(id=task_id, + duration=duration, + priority=priority, + assignments= assignments, + quantity=quantity) + + # Check for micro-batches + if not micro_batch_size: + self.task_dict[task_id] = temp_task # Add task to dictionary + + # Add predecessor to dictionary + self.pred_dict[task_id] = predecessors + else: + self.pred_dict[task_id] = predecessors + batches = self.create_batch(temp_task, micro_batch_size) + self.task_dict.update({task.id: task for task in batches}) + + # Temporarily copy the original predecessors for the new batches + self.pred_dict.update( + {task.id: predecessors for task in batches}) + self.flow_map.update({task.id: { + "parent": parent_collection, + "predecessor": predecessor_collection} for task in batches}) + self.pred_exploded[task_id] = [task.id for task in batches] + + # Organize predecessors for batches + for task in self.task_dict.values(): + self.organize_predecessors(task) + + # Add predecessors + for task in self.task_dict.values(): + self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary + + # Build final task list + self.tasks_list = [value for key, + value in sorted(self.task_dict.items())] + + # Convert periods to time + def int_to_datetime(self, num, start_time): + try: + # Parse the start time string into a datetime object + start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") + + # Add the number of minutes to the start datetime + delta = timedelta(minutes=num) + result_datetime = start_datetime + delta + return result_datetime + + except Exception as e: + print(num) + + +# Open and read the JSON file +with open('inputs/resource.json', 'r') as file: + r_data = json.load(file) + +# Open and read the JSON file +with open('inputs/tasks_all.json', 'r') as file: + t_data = json.load(file) + + +# Open and read the JSON file +with open('inputs/resourcegroups.json', 'r') as file: + rg_data = json.load(file) + + + +prodscheduler = ProdScheduler() +prodscheduler.create_resource_object(r_data) +print("Resource Objects Created.") + +prodscheduler.create_resource_groups(rg_data) +print("Resource Groups Created.") + +prodscheduler.create_task_object(t_data) +print("Task Objects Created.") +print(f"Original Task Length: {len(t_data)} | Post-Batched Length: {len(prodscheduler.task_dict.values())}") + + +solution = Scheduler(prodscheduler.tasks_list, list(prodscheduler.dict_resource.values())).schedule() +print("Solution Created.") + From 45896988cf8d94225fcb9eee9944a1ed61b58803 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 16 Oct 2024 21:42:49 +0800 Subject: [PATCH 13/27] Updates --- nb.ipynb | 71 ++++++++++++++++++++++++++++++++++++++++++++++-- stresstest.ipynb | 29 ++++++++++++++++++-- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/nb.ipynb b/nb.ipynb index 04e8393..888f046 100644 --- a/nb.ipynb +++ b/nb.ipynb @@ -99,9 +99,76 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scheduled 2 of 2 tasks.\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
task_idassigned_resource_idstask_starttask_endresource_intervals
01[1]040([(0, 20), (30, 40)])
12[2]020([(0, 20)])
\n", + "
" + ], + "text/plain": [ + " task_id assigned_resource_ids task_start task_end resource_intervals\n", + "0 1 [1] 0 40 ([(0, 20), (30, 40)])\n", + "1 2 [2] 0 20 ([(0, 20)])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from src.factryengine.models import Resource, Task\n", "from src.factryengine.scheduler.core import Scheduler\n", diff --git a/stresstest.ipynb b/stresstest.ipynb index e0f1769..c9d24ab 100644 --- a/stresstest.ipynb +++ b/stresstest.ipynb @@ -239,9 +239,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resource Objects Created.\n", + "Resource Groups Created.\n", + "Task Objects Created.\n", + "Original Task Length: 6138 | Post-Batched Length: 6138\n" + ] + }, + { + "ename": "ValueError", + "evalue": "min() arg is an empty sequence", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 1\u001b[0m scheduler \u001b[38;5;241m=\u001b[39m ProdScheduler(\n\u001b[0;32m 2\u001b[0m resource_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resource.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 3\u001b[0m resource_group_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resourcegroups.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 4\u001b[0m task_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/tasks_all.json\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 5\u001b[0m )\n\u001b[1;32m----> 7\u001b[0m \u001b[43mscheduler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[1;32mIn[1], line 227\u001b[0m, in \u001b[0;36mProdScheduler.run_scheduler\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTask Objects Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 224\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal Task Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m | Post-Batched Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict\u001b[38;5;241m.\u001b[39mvalues())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 227\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msolution \u001b[38;5;241m=\u001b[39m \u001b[43mScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtasks_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdict_resource\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 228\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSolution Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\core.py:32\u001b[0m, in \u001b[0;36mScheduler.schedule\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 27\u001b[0m heuristic_solver \u001b[38;5;241m=\u001b[39m HeuristicSolver(\n\u001b[0;32m 28\u001b[0m task_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict, resources\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresources, task_order\u001b[38;5;241m=\u001b[39mtask_order\n\u001b[0;32m 29\u001b[0m )\n\u001b[0;32m 31\u001b[0m \u001b[38;5;66;03m# Use the heuristic solver to find a solution\u001b[39;00m\n\u001b[1;32m---> 32\u001b[0m solver_result \u001b[38;5;241m=\u001b[39m \u001b[43mheuristic_solver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;66;03m# Create a scheduler result with the solver result and unscheduled tasks\u001b[39;00m\n\u001b[0;32m 35\u001b[0m scheduler_result \u001b[38;5;241m=\u001b[39m SchedulerResult(\n\u001b[0;32m 36\u001b[0m task_vars\u001b[38;5;241m=\u001b[39msolver_result,\n\u001b[0;32m 37\u001b[0m unscheduled_task_ids\u001b[38;5;241m=\u001b[39mheuristic_solver\u001b[38;5;241m.\u001b[39munscheduled_task_ids,\n\u001b[0;32m 38\u001b[0m )\n", + "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\heuristic_solver\\main.py:85\u001b[0m, in \u001b[0;36mHeuristicSolver.solve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;66;03m# update resource windows\u001b[39;00m\n\u001b[0;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwindow_manager\u001b[38;5;241m.\u001b[39mupdate_resource_windows(allocated_resource_windows_dict)\n\u001b[0;32m 82\u001b[0m task_values \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 83\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: task_id,\n\u001b[0;32m 84\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massigned_resource_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mlist\u001b[39m(allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mkeys()),\n\u001b[1;32m---> 85\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_start\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mallocated_resource_windows_dict\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\n\u001b[0;32m 87\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[0;32m 88\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_end\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mmax\u001b[39m(\n\u001b[0;32m 89\u001b[0m end \u001b[38;5;28;01mfor\u001b[39;00m intervals \u001b[38;5;129;01min\u001b[39;00m allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues() \u001b[38;5;28;01mfor\u001b[39;00m _, end \u001b[38;5;129;01min\u001b[39;00m intervals\n\u001b[0;32m 90\u001b[0m ),\n\u001b[0;32m 91\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresource_intervals\u001b[39m\u001b[38;5;124m\"\u001b[39m: allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues(),\n\u001b[0;32m 92\u001b[0m }\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars[task_id] \u001b[38;5;241m=\u001b[39m task_values\n\u001b[0;32m 96\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(\n\u001b[0;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[0;32m 98\u001b[0m )\n", + "\u001b[1;31mValueError\u001b[0m: min() arg is an empty sequence" + ] + } + ], "source": [ "scheduler = ProdScheduler(\n", " resource_dir = 'inputs/actual_data/resource.json',\n", From 70191f8304aace1a16fd72042fddfe621e37cb02 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Thu, 17 Oct 2024 15:42:23 +0800 Subject: [PATCH 14/27] Task allocator update --- .../heuristic_solver/task_allocator.py | 130 ++++++++++-------- 1 file changed, 73 insertions(+), 57 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 7ad2584..fcc8003 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -31,6 +31,7 @@ def allocate_task( resource_windows_matrix=resource_windows_matrix, task_duration=task_duration, ) + if assignments and constraints: # update the resource matrix with the constraint matrix self._apply_constraint_to_resource_windows_matrix( @@ -162,61 +163,73 @@ def _solve_task_end( return col0_value, other_columns_values def _get_resource_intervals( - self, matrix: Matrix, resource_windows_dict: dict[int, np.ndarray] + self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]] ) -> dict[int, list[tuple[int, int]]]: """ - Extracts the resource intervals from the solution matrix by matching them - with the updated windows provided in `resource_windows_dict`. + Extracts the resource intervals from the solution matrix by strictly matching them + with the provided original windows in `resource_windows_dict`. """ resource_windows_output = {} - # Iterate over each resource ID and its intervals in the solution matrix + # Iterate over each resource ID and its corresponding matrix intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): - indexes = self._find_indexes(resource_intervals.data) - if indexes is not None: - start_index, end_index = indexes + # Retrieve the original windows for the resource + original_windows = resource_windows_dict.get(resource_id, []) - # Extract allocated intervals from the matrix - allocated_intervals = [ - (ceil(round(matrix.intervals[i], 1)), ceil(round(matrix.intervals[i + 1], 1))) - for i in range(start_index, end_index, 2) - ] + # If no windows are found, skip this resource + if len(original_windows) == 0: + print(f"No original windows found for resource {resource_id}") + resource_windows_output[resource_id] = [] + continue - # Get the resource’s available windows from the resource_windows_dict - updated_windows = resource_windows_dict.get(resource_id, []) + # Extract start and end points from the original windows into sets for fast lookups + window_starts = set(window[0] for window in original_windows) + window_ends = set(window[1] for window in original_windows) - # Match allocated intervals with the available windows - matched_intervals = self._match_intervals_with_windows( - allocated_intervals, updated_windows - ) + # Prepare matrix intervals + interval_starts = matrix.intervals[:-1] + interval_ends = matrix.intervals[1:] + + # Vectorized filtering: Keep only intervals where either the start or end matches + mask = np.isin(interval_starts, list(window_starts)) | np.isin(interval_ends, list(window_ends)) + valid_starts = interval_starts[mask] + valid_ends = interval_ends[mask] + + # Combine valid starts and ends into intervals + filtered_intervals = list(zip(np.ceil(valid_starts).astype(int), np.ceil(valid_ends).astype(int))) - # Store the matched intervals in the output dictionary - resource_windows_output[resource_id] = matched_intervals + # Merge contiguous or overlapping intervals + resource_windows_output[resource_id] = self.merge_intervals(filtered_intervals) return resource_windows_output - def _match_intervals_with_windows( - self, allocated_intervals: list[tuple[int, int]], windows: np.ndarray - ) -> list[tuple[int, int]]: + def merge_intervals(self, intervals: list[tuple[int, int]]) -> list[tuple[int, int]]: """ - Matches the allocated intervals with the resource's available windows. + Merges contiguous or overlapping intervals into a single interval. """ - matched_intervals = [] + if not intervals: + return [] + + # Use numpy for fast sorting + intervals = np.array(intervals) + sorted_intervals = intervals[np.argsort(intervals[:, 0])] + + # Initialize merged intervals with the first interval + merged = [sorted_intervals[0]] + + # Vectorized merging + for start, end in sorted_intervals[1:]: + last_start, last_end = merged[-1] - # Iterate over allocated intervals and compare with the available windows - for allocated_start, allocated_end in allocated_intervals: - for window in windows: - window_start, window_end = window["start"], window["end"] + # Merge if overlapping or contiguous + if start <= last_end: + merged[-1] = (last_start, max(last_end, end)) + else: + merged.append((start, end)) - # Check if there is an overlap between the allocated interval and the window - if allocated_end > window_start and allocated_start < window_end: - # Calculate the overlapping interval - matched_start = max(allocated_start, window_start) - matched_end = min(allocated_end, window_end) - matched_intervals.append((matched_start, matched_end)) + return merged - return matched_intervals def _mask_smallest_elements_except_top_k_per_row( @@ -383,47 +396,50 @@ def _create_resource_group_matrix( def _create_constraints_matrix( self, - resource_constraints: list[Resource], + resource_constraints: set[Resource], resource_windows_matrix: Matrix, task_duration: int, ) -> Matrix: """ - Creates a constraints matrix by accumulating availability across multiple windows, - following the structure of the resource group matrix logic. + Checks if the resource constraints are available and updates the resource windows matrix. """ if not resource_constraints: return None - # Extract the resource IDs from the constraints + # get the constraint resource ids resource_ids = np.array([resource.id for resource in resource_constraints]) - # Find the intersection of the resource constraints and the windows matrix - available_resources = np.intersect1d(resource_ids, resource_windows_matrix.resource_ids) - if len(available_resources) == 0: - return None + # check if all resource constraints are available + if not np.all(np.isin(resource_ids, resource_windows_matrix.resource_ids)): + raise AllocationError("All resource constraints are not available") - # Find the indices of the relevant resources in the windows matrix + # Find the indices of the available resources in the windows matrix resource_indexes = np.where( - np.isin(resource_windows_matrix.resource_ids, available_resources) + np.isin(resource_windows_matrix.resource_ids, resource_ids) )[0] - # Extract the relevant windows from the matrix - constraint_matrix = resource_windows_matrix.resource_matrix[:, resource_indexes] - - # Accumulate availability across windows using cumulative sum, resetting at gaps (-1) - accumulated_availability = self._cumsum_reset_at_minus_one_2d(constraint_matrix) + # get the windows for the resource constraints + constraint_windows = resource_windows_matrix.resource_matrix[ + :, resource_indexes + ] - # Check if accumulated availability meets the task duration requirement - if np.sum(accumulated_availability) < task_duration: - raise AllocationError("No solution found: Task duration exceeds available windows.") + # Compute the minimum along axis 1, mask values <= 0, and compute the cumulative sum + # devide by the number of resources to not increase the task completion time + min_values_matrix = ( + np.min(constraint_windows, axis=1, keepdims=True) + * np.ones_like(constraint_windows) + / len(resource_ids) + ) - # Mask zero values to represent unavailable slots - accumulated_availability = np.ma.masked_less_equal(accumulated_availability, 0) + resource_matrix = np.ma.masked_less_equal( + x=min_values_matrix, + value=0, + ).cumsum(axis=0) return Matrix( resource_ids=resource_ids, intervals=resource_windows_matrix.intervals, - resource_matrix=accumulated_availability, + resource_matrix=resource_matrix, ) From 59fcedf0dd601320d3704319739d4a65cab3d368 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Thu, 17 Oct 2024 18:15:24 +0800 Subject: [PATCH 15/27] Committing for branch change --- nb.ipynb | 188 ++++-------------- .../scheduler/heuristic_solver/main.py | 2 + stresstest.ipynb | 51 ++--- 3 files changed, 64 insertions(+), 177 deletions(-) diff --git a/nb.ipynb b/nb.ipynb index 888f046..bb42007 100644 --- a/nb.ipynb +++ b/nb.ipynb @@ -2,77 +2,16 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Scheduled 1 of 1 tasks.\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
task_idassigned_resource_idstask_starttask_endresource_intervals
01[3, 1]050([(0, 20), (30, 50)], [(0, 20), (30, 50)])
\n", - "
" - ], - "text/plain": [ - " task_id assigned_resource_ids task_start task_end \\\n", - "0 1 [3, 1] 0 50 \n", - "\n", - " resource_intervals \n", - "0 ([(0, 20), (30, 50)], [(0, 20), (30, 50)]) " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\n", "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", "from src.factryengine.scheduler.core import Scheduler\n", "\n", - "machine = Resource(id=1, available_windows=[(0, 50)])\n", - "operator1 = Resource(id=2, available_windows=[(0, 20), (30, 50)])\n", + "machine = Resource(id=1, available_windows=[(0, 40)])\n", + "operator1 = Resource(id=2, available_windows=[(0, 40), (30, 50)])\n", "operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n", "\n", "operator_group = ResourceGroup(resources=[operator1, operator2])\n", @@ -84,12 +23,6 @@ " id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n", ")\n", "\n", - "\n", - "# # add machine as a constraint\n", - "# t2 = Task(\n", - "# id=2, duration=10, assignments=[assignment], priority=1, constraints=[machine]\n", - "# )\n", - "\n", "tasks = [t1]\n", "resources = [operator1, operator2, machine]\n", "\n", @@ -99,87 +32,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Scheduled 2 of 2 tasks.\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
task_idassigned_resource_idstask_starttask_endresource_intervals
01[1]040([(0, 20), (30, 40)])
12[2]020([(0, 20)])
\n", - "
" - ], - "text/plain": [ - " task_id assigned_resource_ids task_start task_end resource_intervals\n", - "0 1 [1] 0 40 ([(0, 20), (30, 40)])\n", - "1 2 [2] 0 20 ([(0, 20)])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from src.factryengine.models import Resource, Task\n", "from src.factryengine.scheduler.core import Scheduler\n", "\n", "# Define a resource with multiple fragmented windows\n", "machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n", - "machine2 = Resource(id=2, available_windows=[(0, 20), (60, 80)])\n", + "machine2 = Resource(id=2, available_windows=[(30, 50)])\n", "\n", "# Create tasks with constraints\n", - "t1 = Task(id=1, duration=30, priority=1, constraints=[machine])\n", - "t2 = Task(id=2, duration=20, priority=2, constraints=[machine, machine2])\n", + "t1 = Task(id=1, duration=30, priority=1, constraints=[machine, machine2])\n", + "t2 = Task(id=2, duration=30, priority=2, constraints=[machine, machine2])\n", "\n", "tasks = [t1, t2]\n", "resources = [machine, machine2]\n", @@ -198,22 +64,48 @@ "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n", "from src.factryengine.scheduler.core import Scheduler\n", "\n", - "operator1 = Resource(id=1, available_windows=[(0, 20), (40, 60)])\n", - "operator2 = Resource(id=2, available_windows=[(0, 20), (40, 60)])\n", - "operator3 = Resource(id=3, available_windows=[(0, 30), (50, 60), (80, 150)])\n", + "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 60)])\n", + "operator2 = Resource(id=2, available_windows=[(0, 10), (40, 60)])\n", + "operator3 = Resource(id=3, available_windows=[(10, 20), (50, 60), (80, 150)])\n", "\n", "rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n", "\n", "assignment = Assignment(resource_groups=[rg1], resource_count=1)\n", "\n", "t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n", - "t2 = Task(id=2, duration=20, assignments=[assignment], priority=2)\n", "\n", - "tasks = [t1, t2]\n", + "tasks = [t1]\n", "resources = [operator1, operator2, operator3]\n", "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", "result.to_dict()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n", + "from src.factryengine.scheduler.core import Scheduler\n", + "\n", + "# create the resource\n", + "resource = Resource(id=1, available_windows=[(0, 10), (20, 30)])\n", + "\n", + "# create the resource group\n", + "resource_group = ResourceGroup(resources=[resource])\n", + "\n", + "# create the assignment\n", + "assignment = Assignment(resource_groups=[resource_group], resource_count=1)\n", + "\n", + "# create tasks\n", + "t1 = Task(id=1, duration=5, priority=1, constraints=[resource], predecessor_ids=[2])\n", + "t2 = Task(id=2, duration=5, priority=1, constraints=[resource])\n", + "\n", + "tasks = [t1, t2]\n", + "result = Scheduler(tasks=tasks, resources=[resource]).schedule()\n", + "result.to_dict()" + ] } ], "metadata": { diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py index 5b51052..ce2c5c8 100644 --- a/src/factryengine/scheduler/heuristic_solver/main.py +++ b/src/factryengine/scheduler/heuristic_solver/main.py @@ -71,6 +71,8 @@ def solve(self) -> list[dict]: task_duration=task.duration, constraints=task.constraints, ) + + print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}") except AllocationError as e: self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e)) continue diff --git a/stresstest.ipynb b/stresstest.ipynb index c9d24ab..b62d452 100644 --- a/stresstest.ipynb +++ b/stresstest.ipynb @@ -239,34 +239,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Resource Objects Created.\n", - "Resource Groups Created.\n", - "Task Objects Created.\n", - "Original Task Length: 6138 | Post-Batched Length: 6138\n" - ] - }, - { - "ename": "ValueError", - "evalue": "min() arg is an empty sequence", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[2], line 7\u001b[0m\n\u001b[0;32m 1\u001b[0m scheduler \u001b[38;5;241m=\u001b[39m ProdScheduler(\n\u001b[0;32m 2\u001b[0m resource_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resource.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 3\u001b[0m resource_group_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/resourcegroups.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 4\u001b[0m task_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minputs/actual_data/tasks_all.json\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 5\u001b[0m )\n\u001b[1;32m----> 7\u001b[0m \u001b[43mscheduler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[1;32mIn[1], line 227\u001b[0m, in \u001b[0;36mProdScheduler.run_scheduler\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 223\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTask Objects Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 224\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal Task Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mt_data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m | Post-Batched Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict\u001b[38;5;241m.\u001b[39mvalues())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 227\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msolution \u001b[38;5;241m=\u001b[39m \u001b[43mScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtasks_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdict_resource\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 228\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSolution Created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\core.py:32\u001b[0m, in \u001b[0;36mScheduler.schedule\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 27\u001b[0m heuristic_solver \u001b[38;5;241m=\u001b[39m HeuristicSolver(\n\u001b[0;32m 28\u001b[0m task_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_dict, resources\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresources, task_order\u001b[38;5;241m=\u001b[39mtask_order\n\u001b[0;32m 29\u001b[0m )\n\u001b[0;32m 31\u001b[0m \u001b[38;5;66;03m# Use the heuristic solver to find a solution\u001b[39;00m\n\u001b[1;32m---> 32\u001b[0m solver_result \u001b[38;5;241m=\u001b[39m \u001b[43mheuristic_solver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;66;03m# Create a scheduler result with the solver result and unscheduled tasks\u001b[39;00m\n\u001b[0;32m 35\u001b[0m scheduler_result \u001b[38;5;241m=\u001b[39m SchedulerResult(\n\u001b[0;32m 36\u001b[0m task_vars\u001b[38;5;241m=\u001b[39msolver_result,\n\u001b[0;32m 37\u001b[0m unscheduled_task_ids\u001b[38;5;241m=\u001b[39mheuristic_solver\u001b[38;5;241m.\u001b[39munscheduled_task_ids,\n\u001b[0;32m 38\u001b[0m )\n", - "File \u001b[1;32mc:\\Projects\\factryengine\\src\\factryengine\\scheduler\\heuristic_solver\\main.py:85\u001b[0m, in \u001b[0;36mHeuristicSolver.solve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;66;03m# update resource windows\u001b[39;00m\n\u001b[0;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwindow_manager\u001b[38;5;241m.\u001b[39mupdate_resource_windows(allocated_resource_windows_dict)\n\u001b[0;32m 82\u001b[0m task_values \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 83\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: task_id,\n\u001b[0;32m 84\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massigned_resource_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mlist\u001b[39m(allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mkeys()),\n\u001b[1;32m---> 85\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_start\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mmin\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mallocated_resource_windows_dict\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mintervals\u001b[49m\n\u001b[0;32m 87\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[0;32m 88\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtask_end\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mmax\u001b[39m(\n\u001b[0;32m 89\u001b[0m end \u001b[38;5;28;01mfor\u001b[39;00m intervals \u001b[38;5;129;01min\u001b[39;00m allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues() \u001b[38;5;28;01mfor\u001b[39;00m _, end \u001b[38;5;129;01min\u001b[39;00m intervals\n\u001b[0;32m 90\u001b[0m ),\n\u001b[0;32m 91\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresource_intervals\u001b[39m\u001b[38;5;124m\"\u001b[39m: allocated_resource_windows_dict\u001b[38;5;241m.\u001b[39mvalues(),\n\u001b[0;32m 92\u001b[0m }\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars[task_id] \u001b[38;5;241m=\u001b[39m task_values\n\u001b[0;32m 96\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(\n\u001b[0;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtask_vars\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[0;32m 98\u001b[0m )\n", - "\u001b[1;31mValueError\u001b[0m: min() arg is an empty sequence" - ] - } - ], + "outputs": [], "source": [ "scheduler = ProdScheduler(\n", " resource_dir = 'inputs/actual_data/resource.json',\n", @@ -279,11 +254,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "result = scheduler.solution.to_dataframe()" + "scheduler.dict_resource[119]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scheduler.task_dict['WO138769-10']" ] }, { @@ -326,6 +310,15 @@ "intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n", "intervals.sort_values(by='interval_end', ascending=False)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sched" + ] } ], "metadata": { From 81119fb5d6232caa3f1c2aa01de4f53a4de02845 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Thu, 17 Oct 2024 22:00:05 +0800 Subject: [PATCH 16/27] Used matrices for getting resource intervals --- .../heuristic_solver/task_allocator.py | 178 ++++++++++-------- 1 file changed, 99 insertions(+), 79 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index fcc8003..33e0f92 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -166,71 +166,92 @@ def _get_resource_intervals( self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]] ) -> dict[int, list[tuple[int, int]]]: """ - Extracts the resource intervals from the solution matrix by strictly matching them - with the provided original windows in `resource_windows_dict`. + Extracts all the resource intervals used from the solution matrix, + including non-contiguous intervals and partial usage. """ resource_windows_output = {} - # Iterate over each resource ID and its corresponding matrix intervals + # Iterate over each resource and its corresponding matrix intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): + print(f"\nResource ID: {resource_id}") + print(f"Resource Intervals: {resource_intervals}") + print(f"Overall Intervals: {matrix.intervals}") - # Retrieve the original windows for the resource - original_windows = resource_windows_dict.get(resource_id, []) + # Get all relevant indexes + indexes = self._find_indexes(resource_intervals) + print(f"Produced Indexes: {indexes}") - # If no windows are found, skip this resource - if len(original_windows) == 0: - print(f"No original windows found for resource {resource_id}") - resource_windows_output[resource_id] = [] - continue - - # Extract start and end points from the original windows into sets for fast lookups - window_starts = set(window[0] for window in original_windows) - window_ends = set(window[1] for window in original_windows) - - # Prepare matrix intervals - interval_starts = matrix.intervals[:-1] - interval_ends = matrix.intervals[1:] + # Pair the indexes in groups of 2 (start, end) + intervals = [] + for start, end in zip(indexes[::2], indexes[1::2]): + # Use start and end indexes directly without skipping + print(f"Start: {start}, End: {end}") + interval_start = matrix.intervals[start] + interval_end = matrix.intervals[end] - # Vectorized filtering: Keep only intervals where either the start or end matches - mask = np.isin(interval_starts, list(window_starts)) | np.isin(interval_ends, list(window_ends)) - valid_starts = interval_starts[mask] - valid_ends = interval_ends[mask] + # Append the interval to the list + intervals.append((int(np.round(interval_start)), int(np.round(interval_end)))) - # Combine valid starts and ends into intervals - filtered_intervals = list(zip(np.ceil(valid_starts).astype(int), np.ceil(valid_ends).astype(int))) - - # Merge contiguous or overlapping intervals - resource_windows_output[resource_id] = self.merge_intervals(filtered_intervals) + # Store the intervals for the current resource + resource_windows_output[resource_id] = intervals return resource_windows_output - def merge_intervals(self, intervals: list[tuple[int, int]]) -> list[tuple[int, int]]: + + def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: """ - Merges contiguous or overlapping intervals into a single interval. + Finds the first index where the value is masked, and the next value is non-masked and > 0. """ - if not intervals: - return [] + indexes = [] + # Shift the mask by 1 to align with the 'next' element comparison + current_mask = resource_intervals.mask[:-1] + next_mask = resource_intervals.mask[1:] + next_values = resource_intervals.data[1:] - # Use numpy for fast sorting - intervals = np.array(intervals) - sorted_intervals = intervals[np.argsort(intervals[:, 0])] + # Vectorized condition: current is masked, next is non-masked, and next value > 0 + condition = (current_mask) & (~next_mask) & (next_values > 0) - # Initialize merged intervals with the first interval - merged = [sorted_intervals[0]] + # Find the first index where the condition is met + indices = np.where(condition)[0] - # Vectorized merging - for start, end in sorted_intervals[1:]: - last_start, last_end = merged[-1] + first_index = indices[0] + last_index = resource_intervals.size-1 - # Merge if overlapping or contiguous - if start <= last_end: - merged[-1] = (last_start, max(last_end, end)) - else: - merged.append((start, end)) + indexes = [first_index] # Start with the first index - return merged + # Iterate through the range between first and last index + for i in range(first_index, last_index + 1): + current = resource_intervals[i] + previous = resource_intervals[i - 1] if i > 0 else 0 + next_value = resource_intervals[i + 1] if i < last_index else 0 + # Check if the current value is masked + is_masked = resource_intervals.mask[i - 1] if i > 0 else False + # Skip if all values are the same (stable window) + if current > 0 and current == previous == next_value: + continue + + # Skip increasing trend from masked value + if current > 0 and current < next_value and is_masked: + continue + + # Detect end of a window + if current > 0 and current == next_value and (is_masked or previous < current): + indexes.append(i) + continue + + # Detect start of a new window + if current > 0 and next_value > current and (is_masked or previous == current): + indexes.append(i) + continue + + # Always add the last index + if i == last_index: + indexes.append(i) + + # Return the first valid index, or None if no valid index is found + return indexes def _mask_smallest_elements_except_top_k_per_row( self, array: np.ma.core.MaskedArray, k @@ -502,48 +523,47 @@ def _create_assignments_matrix( return assignments_matrix + # def _find_indexes(self, arr: np.array) -> tuple[int, int] | None: + # """ + # Find the start and end indexes for a valid segment of resource availability. + # This version avoids explicit loops and ensures the start index is correctly identified. + # """ + # # If the input is a MaskedArray, handle it accordingly + # if isinstance(arr, np.ma.MaskedArray): + # arr_data = arr.data + # mask = arr.mask + # # Find valid (unmasked and positive) indices + # valid_indices = np.where((~mask) & (arr_data >= 0))[0] + # else: + # valid_indices = np.where(arr >= 0)[0] - def _find_indexes(self, arr: np.array) -> tuple[int, int] | None: - """ - Find the start and end indexes for a valid segment of resource availability. - This version avoids explicit loops and ensures the start index is correctly identified. - """ - # If the input is a MaskedArray, handle it accordingly - if isinstance(arr, np.ma.MaskedArray): - arr_data = arr.data - mask = arr.mask - # Find valid (unmasked and positive) indices - valid_indices = np.where((~mask) & (arr_data >= 0))[0] - else: - valid_indices = np.where(arr >= 0)[0] - - # If no valid indices are found, return None (no available resources) - if valid_indices.size == 0: - return None + # # If no valid indices are found, return None (no available resources) + # if valid_indices.size == 0: + # return None - # Identify if the start of the array is valid - start_index = 0 if arr[0] > 0 else valid_indices[0] + # # Identify if the start of the array is valid + # start_index = 0 if arr[0] > 0 else valid_indices[0] - # Calculate differences between consecutive indices - diffs = np.diff(valid_indices) + # # Calculate differences between consecutive indices + # diffs = np.diff(valid_indices) - # Identify segment boundaries where there is a gap greater than 1 - gaps = diffs > 1 - segment_boundaries = np.where(gaps)[0] + # # Identify segment boundaries where there is a gap greater than 1 + # gaps = diffs > 1 + # segment_boundaries = np.where(gaps)[0] - # Insert the start index explicitly to ensure it is considered - segment_starts = np.insert(segment_boundaries + 1, 0, 0) - segment_ends = np.append(segment_starts[1:], len(valid_indices)) + # # Insert the start index explicitly to ensure it is considered + # segment_starts = np.insert(segment_boundaries + 1, 0, 0) + # segment_ends = np.append(segment_starts[1:], len(valid_indices)) - # Always take the first segment (which starts at the earliest valid index) - start_pos = segment_starts[0] - end_pos = segment_ends[0] - 1 + # # Always take the first segment (which starts at the earliest valid index) + # start_pos = segment_starts[0] + # end_pos = segment_ends[0] - 1 - # Convert these segment positions to the actual start and end indices - start_index = valid_indices[start_pos] - end_index = valid_indices[end_pos] + # # Convert these segment positions to the actual start and end indices + # start_index = valid_indices[start_pos] + # end_index = valid_indices[end_pos] - return start_index, end_index + # return start_index, end_index From e8eee2291473ac92e2fb4a8a6d716d3c81426e01 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Thu, 17 Oct 2024 22:21:34 +0800 Subject: [PATCH 17/27] Stable scheduler --- src/factryengine/scheduler/heuristic_solver/main.py | 2 -- .../scheduler/heuristic_solver/task_allocator.py | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/main.py b/src/factryengine/scheduler/heuristic_solver/main.py index ce2c5c8..5b51052 100644 --- a/src/factryengine/scheduler/heuristic_solver/main.py +++ b/src/factryengine/scheduler/heuristic_solver/main.py @@ -71,8 +71,6 @@ def solve(self) -> list[dict]: task_duration=task.duration, constraints=task.constraints, ) - - print(f"Allocated resources for task {task_id}: {allocated_resource_windows_dict}") except AllocationError as e: self.mark_task_as_unscheduled(task_id=task_id, error_message=str(e)) continue diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 33e0f92..3f18f05 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -173,19 +173,19 @@ def _get_resource_intervals( # Iterate over each resource and its corresponding matrix intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): - print(f"\nResource ID: {resource_id}") - print(f"Resource Intervals: {resource_intervals}") - print(f"Overall Intervals: {matrix.intervals}") + # print(f"\nResource ID: {resource_id}") + # print(f"Resource Intervals: {resource_intervals}") + # print(f"Overall Intervals: {matrix.intervals}") # Get all relevant indexes indexes = self._find_indexes(resource_intervals) - print(f"Produced Indexes: {indexes}") + # print(f"Produced Indexes: {indexes}") # Pair the indexes in groups of 2 (start, end) intervals = [] for start, end in zip(indexes[::2], indexes[1::2]): # Use start and end indexes directly without skipping - print(f"Start: {start}, End: {end}") + # print(f"Start: {start}, End: {end}") interval_start = matrix.intervals[start] interval_end = matrix.intervals[end] @@ -214,7 +214,7 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: # Find the first index where the condition is met indices = np.where(condition)[0] - first_index = indices[0] + first_index = indices[0] if len(indices) > 0 else 0 last_index = resource_intervals.size-1 indexes = [first_index] # Start with the first index From d30f06a14c17ecbbabd646e3ec83b597e0369c6d Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 21 Oct 2024 20:25:50 +0800 Subject: [PATCH 18/27] Updated get_resource_intervals test --- tests/scheduler/test_task_allocator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index 0a7a71f..75571f4 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -31,8 +31,8 @@ def test_solve_task_end(task_allocator): def test_get_resource_intervals_continuous(task_allocator): # Test case continuous values 1 task 2 resources solution_resource_ids = np.array([1, 2]) - solution_intervals = np.array([0, 1, 2, 3]) - resource_matrix = np.ma.array([[0, 0], [1, 1]]) + solution_intervals = np.array([0, 1]) + resource_matrix = np.ma.array([[0, 0], [1, 1]], mask=[[False, False], [False, False]],) solution_matrix = Matrix( resource_ids=solution_resource_ids, intervals=solution_intervals, @@ -45,15 +45,15 @@ def test_get_resource_intervals_continuous(task_allocator): def test_get_resource_intervals_windowed(task_allocator): # Test case windowed values 1 task 1 resource solution_resource_ids = np.array([1]) - solution_intervals = np.array([0, 1, 4, 5, 7, 8]) - resource_matrix = np.ma.array([[0], [1], [4], [5], [7], [8]]) + solution_intervals = np.array([0, 2, 3, 4]) + resource_matrix = np.ma.array([[0], [2], [2], [3]], mask=[[False], [False], [False], [False]],) solution_matrix = Matrix( resource_ids=solution_resource_ids, intervals=solution_intervals, resource_matrix=resource_matrix, ) result = task_allocator._get_resource_intervals(solution_matrix) - expeceted = {1: [(0, 1), (4, 5), (7, 8)]} + expeceted = {1: [(0, 2), (3, 4)]} assert result == expeceted From 3b2df6f9191f0fc9fda51694b526cc3e2171fc7a Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 21 Oct 2024 20:35:38 +0800 Subject: [PATCH 19/27] Updated find indexes test --- tests/scheduler/test_task_allocator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index 75571f4..c6785b2 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -220,9 +220,8 @@ def test_diff_and_zero_negatives(array, expected): "array, expected", [ # Full valid sequence without gaps - (np.array([0, 1, 2, 3, 4]), (0, 4)), - # Sequence with repeated values, expecting first valid segment - (np.array([0, 3]), (0, 1)), # This one might need revisiting if logic changes + (np.ma.array([0, 2, 2, 3], mask=[False, False, False, False]), ([0, 1, 2, 3])), + (np.ma.array([0, 3], mask=[False, False]), ([0, 1])), ], ) def test_find_indexes(array, expected): From 5964f6cc4108284a43d28e179ea83965183a2690 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 21 Oct 2024 20:44:12 +0800 Subject: [PATCH 20/27] Updated test _cumsum_reset_at_minus_one --- tests/scheduler/test_task_allocator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index c6785b2..d6336b3 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row( @pytest.mark.parametrize( "array, expected", [ - (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]), + (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]), (np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]), (np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]), ], From 1f785ccb896a831e00e2fec0d718ae85b6a4e0a3 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Mon, 21 Oct 2024 20:48:17 +0800 Subject: [PATCH 21/27] Reverted cumsum at minus -1 test --- tests/scheduler/test_task_allocator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index d6336b3..c6785b2 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row( @pytest.mark.parametrize( "array, expected", [ - (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]), + (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]), (np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]), (np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]), ], From 22dc0a7f3de26f4ed485f3ac2ec3701e50d1ab25 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 22 Oct 2024 14:02:58 +0800 Subject: [PATCH 22/27] Updates to test and task allocator --- .../heuristic_solver/task_allocator.py | 63 ++++++++++++------- tests/scheduler/test_task_allocator.py | 2 +- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index 3f18f05..c295fde 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -45,29 +45,41 @@ def allocate_task( task_duration=task_duration, ) - # matrix to solve - matrix_to_solve = assignments_matrix or constraints_matrix + if assignments_matrix and constraints_matrix: + # find the solution for assignments + solution_matrix = self._solve_matrix( + matrix=assignments_matrix, + task_duration=task_duration, + ) + + # find the solution for constraints + constraints_solution = self._solve_matrix( + matrix=constraints_matrix, + task_duration=task_duration, + ) + else: + # matrix to solve + matrix_to_solve = assignments_matrix or constraints_matrix + + # find the solution + solution_matrix = self._solve_matrix( + matrix=matrix_to_solve, + task_duration=task_duration, + ) - # find the solution - solution_matrix = self._solve_matrix( - matrix=matrix_to_solve, - task_duration=task_duration, - ) # process solution to find allocated resource windows allocated_windows = self._get_resource_intervals( - matrix=solution_matrix, - resource_windows_dict=resource_windows_dict + matrix=solution_matrix ) # add constraints to allocated windows if constraints and assignments: constraints_matrix_trimmed = Matrix.trim_end( - original_matrix=constraints_matrix, trim_matrix=solution_matrix + original_matrix=constraints_solution, trim_matrix=solution_matrix ) allocated_windows.update( self._get_resource_intervals( - matrix=constraints_matrix_trimmed, - resource_windows_dict=resource_windows_dict + matrix=constraints_matrix_trimmed ) ) @@ -163,7 +175,7 @@ def _solve_task_end( return col0_value, other_columns_values def _get_resource_intervals( - self, matrix: Matrix, resource_windows_dict: dict[int, list[tuple[float, float, float, int]]] + self, matrix: Matrix ) -> dict[int, list[tuple[int, int]]]: """ Extracts all the resource intervals used from the solution matrix, @@ -173,13 +185,10 @@ def _get_resource_intervals( # Iterate over each resource and its corresponding matrix intervals for resource_id, resource_intervals in zip(matrix.resource_ids, matrix.resource_matrix.T): - # print(f"\nResource ID: {resource_id}") - # print(f"Resource Intervals: {resource_intervals}") - # print(f"Overall Intervals: {matrix.intervals}") + # Get all relevant indexes indexes = self._find_indexes(resource_intervals) - # print(f"Produced Indexes: {indexes}") # Pair the indexes in groups of 2 (start, end) intervals = [] @@ -226,23 +235,35 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: next_value = resource_intervals[i + 1] if i < last_index else 0 # Check if the current value is masked - is_masked = resource_intervals.mask[i - 1] if i > 0 else False + is_prev_masked = resource_intervals.mask[i - 1] if i > 0 else False + is_curr_masked = resource_intervals.mask[i] + is_next_masked = resource_intervals.mask[i+1] if i < last_index else False # Skip if all values are the same (stable window) if current > 0 and current == previous == next_value: continue # Skip increasing trend from masked value - if current > 0 and current < next_value and is_masked: + if current > 0 and current < next_value and is_prev_masked: continue # Detect end of a window - if current > 0 and current == next_value and (is_masked or previous < current): + if current > 0 and current == next_value and (is_prev_masked or previous < current): + indexes.append(i) + continue + + # Detect end of window using masks + if current > 0 and is_next_masked and not is_curr_masked: indexes.append(i) continue # Detect start of a new window - if current > 0 and next_value > current and (is_masked or previous == current): + if current > 0 and next_value > current and (is_prev_masked or previous == current): + indexes.append(i) + continue + + # Detect start of window using masks + if is_curr_masked and previous > 0 and next_value > 0: indexes.append(i) continue diff --git a/tests/scheduler/test_task_allocator.py b/tests/scheduler/test_task_allocator.py index c6785b2..d6336b3 100644 --- a/tests/scheduler/test_task_allocator.py +++ b/tests/scheduler/test_task_allocator.py @@ -126,7 +126,7 @@ def test_mask_smallest_elements_except_top_k_per_row( @pytest.mark.parametrize( "array, expected", [ - (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 10]), + (np.array([0, 1, 5, -1, 10]), [0, 1, 6, 0, 16]), (np.array([-1, 2, 3, 0, 4]), [0, 2, 5, 5, 9]), (np.array([0, -1, 2, 4, -1]), [0, 0, 2, 6, 0]), ], From 2ac6f16a0dc3c781625d0d5472adb2f19412fdc2 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 22 Oct 2024 16:44:13 +0800 Subject: [PATCH 23/27] Updated task allocator comments --- src/factryengine/scheduler/heuristic_solver/task_allocator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index c295fde..c2afa4c 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -209,7 +209,7 @@ def _get_resource_intervals( def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: """ - Finds the first index where the value is masked, and the next value is non-masked and > 0. + Finds relevant indexes in the resource intervals where the resource is used. """ indexes = [] # Shift the mask by 1 to align with the 'next' element comparison From 4a31734cf7dd206e9b2dee546ebbecbc62562560 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 22 Oct 2024 16:44:51 +0800 Subject: [PATCH 24/27] Ignored files with .ipynb extensions --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d8d9708..85f38dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Prototyping Notebooks notebooks/ +*.ipynb # Test data inputs/ From 610d132cc303c34d2504aeb4d5a297ca539b836e Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 22 Oct 2024 16:45:40 +0800 Subject: [PATCH 25/27] Move files to notebooks folder --- nb.ipynb | 132 ------------------ stresstest.ipynb | 345 ----------------------------------------------- 2 files changed, 477 deletions(-) delete mode 100644 nb.ipynb delete mode 100644 stresstest.ipynb diff --git a/nb.ipynb b/nb.ipynb deleted file mode 100644 index bb42007..0000000 --- a/nb.ipynb +++ /dev/null @@ -1,132 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", - "from src.factryengine.scheduler.core import Scheduler\n", - "\n", - "machine = Resource(id=1, available_windows=[(0, 40)])\n", - "operator1 = Resource(id=2, available_windows=[(0, 40), (30, 50)])\n", - "operator2 = Resource(id=3, available_windows=[(0, 20), (30, 50)])\n", - "\n", - "operator_group = ResourceGroup(resources=[operator1, operator2])\n", - "\n", - "assignment = Assignment(resource_groups=[operator_group], resource_count=1)\n", - "\n", - "# add machine as a constraint\n", - "t1 = Task(\n", - " id=1, duration=40, assignments=[assignment], priority=1, constraints=[machine]\n", - ")\n", - "\n", - "tasks = [t1]\n", - "resources = [operator1, operator2, machine]\n", - "\n", - "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", - "result.to_dataframe()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from src.factryengine.models import Resource, Task\n", - "from src.factryengine.scheduler.core import Scheduler\n", - "\n", - "# Define a resource with multiple fragmented windows\n", - "machine = Resource(id=1, available_windows=[(0, 20), (30, 50)])\n", - "machine2 = Resource(id=2, available_windows=[(30, 50)])\n", - "\n", - "# Create tasks with constraints\n", - "t1 = Task(id=1, duration=30, priority=1, constraints=[machine, machine2])\n", - "t2 = Task(id=2, duration=30, priority=2, constraints=[machine, machine2])\n", - "\n", - "tasks = [t1, t2]\n", - "resources = [machine, machine2]\n", - "\n", - "# Schedule the tasks\n", - "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", - "result.to_dataframe()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n", - "from src.factryengine.scheduler.core import Scheduler\n", - "\n", - "operator1 = Resource(id=1, available_windows=[(0, 10), (40, 60)])\n", - "operator2 = Resource(id=2, available_windows=[(0, 10), (40, 60)])\n", - "operator3 = Resource(id=3, available_windows=[(10, 20), (50, 60), (80, 150)])\n", - "\n", - "rg1 = ResourceGroup(resources=[operator1, operator2, operator3])\n", - "\n", - "assignment = Assignment(resource_groups=[rg1], resource_count=1)\n", - "\n", - "t1 = Task(id=1, duration=40, assignments=[assignment], priority=1)\n", - "\n", - "tasks = [t1]\n", - "resources = [operator1, operator2, operator3]\n", - "result = Scheduler(tasks=tasks, resources=resources).schedule()\n", - "result.to_dict()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from src.factryengine.models import Resource, ResourceGroup, Task, Assignment\n", - "from src.factryengine.scheduler.core import Scheduler\n", - "\n", - "# create the resource\n", - "resource = Resource(id=1, available_windows=[(0, 10), (20, 30)])\n", - "\n", - "# create the resource group\n", - "resource_group = ResourceGroup(resources=[resource])\n", - "\n", - "# create the assignment\n", - "assignment = Assignment(resource_groups=[resource_group], resource_count=1)\n", - "\n", - "# create tasks\n", - "t1 = Task(id=1, duration=5, priority=1, constraints=[resource], predecessor_ids=[2])\n", - "t2 = Task(id=2, duration=5, priority=1, constraints=[resource])\n", - "\n", - "tasks = [t1, t2]\n", - "result = Scheduler(tasks=tasks, resources=[resource]).schedule()\n", - "result.to_dict()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/stresstest.ipynb b/stresstest.ipynb deleted file mode 100644 index b62d452..0000000 --- a/stresstest.ipynb +++ /dev/null @@ -1,345 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import json\n", - "import pytz\n", - "from datetime import datetime, timezone, timedelta\n", - "import time\n", - "from src.factryengine.models import Resource, Task, Assignment, ResourceGroup\n", - "from src.factryengine.scheduler.core import Scheduler\n", - "from src.factryengine.scheduler.task_batch_processor import TaskSplitter\n", - "\n", - "\n", - "class ProdScheduler: \n", - " def __init__(self, \n", - " resource_dir: str, \n", - " resource_group_dir: str, \n", - " task_dir: str\n", - " ) -> None:\n", - " # Scheduler Attributes\n", - " self.cph_timezone = pytz.timezone('Europe/Copenhagen')\n", - " # self.today = datetime.now(timezone.utc).replace(\n", - " # hour=0, minute=0, second=0, microsecond=0)\n", - " self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc)\n", - " self.today_str = str(self.today)[:19]\n", - "\n", - " # Component Attributes\n", - " self.dict_resource = {}\n", - " self.dict_resourcegroups = {}\n", - " self.tasks_list = []\n", - " self.task_dict = {}\n", - " self.pred_dict = {}\n", - " self.flow_map = {}\n", - " self.pred_exploded = {}\n", - "\n", - " # Inputs \n", - " with open(resource_dir, 'r') as file:\n", - " self.r_data = json.load(file)\n", - "\n", - " with open(task_dir, 'r') as file:\n", - " self.t_data = json.load(file)\n", - "\n", - " with open(resource_group_dir, 'r') as file:\n", - " self.rg_data = json.load(file)\n", - " \n", - " def convert_to_minutes(self, datetime_str, start_time_obj):\n", - " datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z')\n", - " diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60\n", - " return int(diff_minutes)\n", - "\n", - " def adjust_capacity(self, start, end, capacity):\n", - " return (end - start) * capacity + start\n", - "\n", - " def organize_predecessors(self, task: Task):\n", - " try:\n", - " list_predecessors = self.pred_dict[task.id]\n", - " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n", - " if task.batch_id: # Check if task is microbatched\n", - " # Look for each predecessors that exist in the flow map\n", - " for predecessor in list_predecessors:\n", - " pred_batch_id = f'{predecessor}-{task.batch_id}'\n", - " if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow\n", - " # Check if pred-parent connection is correct\n", - " if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']:\n", - " self.pred_dict[task.id] = [pred_batch_id]\n", - "\n", - " elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict:\n", - " parent_predecessor = []\n", - " for pred in self.pred_dict[task.id]:\n", - " parent_predecessor.extend(self.pred_dict[pred])\n", - " self.pred_dict[task.id] = parent_predecessor\n", - " # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW\n", - "\n", - " # Remove task batch id\n", - " task.set_batch_id(None)\n", - "\n", - " # Check for predecessors to be exploded\n", - " for predecessor in list_predecessors:\n", - " if predecessor in self.pred_exploded:\n", - " self.pred_dict[task.id].remove(\n", - " predecessor) # Remove original value\n", - " self.pred_dict[task.id].extend(\n", - " self.pred_exploded[predecessor]) # Add exploded batches\n", - " except Exception as e:\n", - " return\n", - " \n", - " def set_predecessors(self, task: Task):\n", - " if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors\n", - " return\n", - "\n", - " for pred_id in self.pred_dict[task.id]:\n", - " if pred_id in self.task_dict: # ensure predecessor exists in task_dict\n", - " pred_task = self.task_dict[pred_id]\n", - "\n", - " # Avoid adding a predecessor multiple times\n", - " if pred_task not in task.predecessors:\n", - " # set predecessors for the predecessor first\n", - " self.set_predecessors(pred_task)\n", - " task.predecessors.append(pred_task)\n", - " \n", - " def create_resource_object(self, resource_list):\n", - " # Generate slot based on schedule selected in NocoDB\n", - " for row in resource_list:\n", - " periods_list = []\n", - " for sched in row['availability']:\n", - " if not sched['is_absent']:\n", - " start = self.convert_to_minutes(\n", - " sched['start_datetime'], self.today)\n", - " end = self.convert_to_minutes(\n", - " sched['end_datetime'], self.today)\n", - " # ========= Uncomment to use capacity\n", - " capacity = sched['capacity_percent']\n", - " # capacity = None\n", - " periods_list.append((int(start), int(self.adjust_capacity(\n", - " start, end, capacity)) if capacity else int(end)))\n", - "\n", - " resource_id = int(row['resource_id'])\n", - " self.dict_resource[resource_id] = Resource(\n", - " id=resource_id, available_windows=periods_list)\n", - "\n", - " def create_resource_groups(self, resource_group_list):\n", - " # Generate Resource Groups\n", - " for x in resource_group_list:\n", - " resource_list = []\n", - " resources = x['resource_id']\n", - " for r in resources:\n", - " if r in self.dict_resource:\n", - " resource_list.append(self.dict_resource[r])\n", - " \n", - " if resource_list:\n", - " self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup(\n", - " id=x['resource_group_id'], resources=resource_list)\n", - "\n", - " def create_batch(self, task: Task, batch_size: int):\n", - " batches = TaskSplitter(task, batch_size).split_into_batches()\n", - " counter = 1\n", - " for batch in batches:\n", - " batch.id = f\"{task.id}-{counter}\"\n", - " counter += 1\n", - "\n", - " return batches\n", - "\n", - " def create_task_object(self, task_list):\n", - " for i in task_list:\n", - " rg_list = []\n", - " task_id = i['taskno']\n", - " duration = int(i['duration'])\n", - " priority = int(i['priority'])\n", - " quantity = int(i['quantity'])\n", - " # micro_batch_size = int(\n", - " # i['micro_batch_size']) if i['micro_batch_size'] else None\n", - " micro_batch_size = None\n", - " resource_group_id = i['resource_group_id']\n", - " rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups]\n", - " predecessors = i['predecessors']\n", - " parent_collection = i['parent_item_collection_id'] if micro_batch_size else None\n", - " predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None\n", - "\n", - " assignments = []\n", - " # Create assignments \n", - " for x in resource_group_id: \n", - " if x in self.dict_resourcegroups:\n", - " assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1))\n", - " \n", - " # Temporarily add into component dicts\n", - " temp_task = Task(id=task_id,\n", - " duration=duration,\n", - " priority=priority,\n", - " assignments= assignments,\n", - " quantity=quantity)\n", - " \n", - " # Check for micro-batches\n", - " if not micro_batch_size:\n", - " self.task_dict[task_id] = temp_task # Add task to dictionary\n", - "\n", - " # Add predecessor to dictionary\n", - " self.pred_dict[task_id] = predecessors\n", - " else:\n", - " self.pred_dict[task_id] = predecessors\n", - " batches = self.create_batch(temp_task, micro_batch_size)\n", - " self.task_dict.update({task.id: task for task in batches})\n", - "\n", - " # Temporarily copy the original predecessors for the new batches\n", - " self.pred_dict.update(\n", - " {task.id: predecessors for task in batches})\n", - " self.flow_map.update({task.id: {\n", - " \"parent\": parent_collection,\n", - " \"predecessor\": predecessor_collection} for task in batches})\n", - " self.pred_exploded[task_id] = [task.id for task in batches]\n", - "\n", - "\n", - " # Organize predecessors for batches\n", - " for task in self.task_dict.values():\n", - " self.organize_predecessors(task)\n", - "\n", - " # Add predecessors\n", - " for task in self.task_dict.values():\n", - " self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary\n", - "\n", - " # Build final task list\n", - " self.tasks_list = [value for key,\n", - " value in sorted(self.task_dict.items())]\n", - " \n", - " # Convert periods to time\n", - " def int_to_datetime(self, num, start_time):\n", - " try:\n", - " # Parse the start time string into a datetime object\n", - " start_datetime = datetime.strptime(start_time, \"%Y-%m-%d %H:%M:%S\")\n", - " \n", - " # Add the number of minutes to the start datetime\n", - " delta = timedelta(minutes=num)\n", - " result_datetime = start_datetime + delta\n", - " return result_datetime\n", - " \n", - " except Exception as e: \n", - " print(num)\n", - " \n", - " def run_scheduler(self):\n", - " self.create_resource_object(self.r_data)\n", - " print(\"Resource Objects Created.\")\n", - "\n", - " self.create_resource_groups(self.rg_data)\n", - " print(\"Resource Groups Created.\")\n", - "\n", - " self.create_task_object(self.t_data)\n", - " print(\"Task Objects Created.\")\n", - " print(f\"Original Task Length: {len(self.t_data)} | Post-Batched Length: {len(self.task_dict.values())}\")\n", - "\n", - "\n", - " self.solution = Scheduler(self.tasks_list, list(self.dict_resource.values())).schedule()\n", - " print(\"Solution Created.\")\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scheduler = ProdScheduler(\n", - " resource_dir = 'inputs/actual_data/resource.json',\n", - " resource_group_dir = 'inputs/actual_data/resourcegroups.json',\n", - " task_dir = 'inputs/actual_data/tasks_all.json'\n", - ")\n", - "\n", - "scheduler.run_scheduler()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scheduler.dict_resource[119]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scheduler.task_dict['WO138769-10']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result['start_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_start'], scheduler.today_str), axis=1)\n", - "result['end_dt'] = result.apply(lambda x: scheduler.int_to_datetime(x['task_end'], scheduler.today_str), axis=1)\n", - "result.sort_values(by='task_end', ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result[result['task_id'].str.startswith('WO137709')].sort_values(by='task_start')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scheduler.task_dict['WO135483-40']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "intervals = scheduler.solution.get_resource_intervals_df()\n", - "intervals['start_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_start'], scheduler.today_str), axis=1)\n", - "intervals['end_dt'] = intervals.apply(lambda x: scheduler.int_to_datetime(x['interval_end'], scheduler.today_str), axis=1)\n", - "intervals.sort_values(by='interval_end', ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sched" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 464aa798251b0b17ad1f2bb6529cdca9b6140c15 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Tue, 22 Oct 2024 16:57:19 +0800 Subject: [PATCH 26/27] Deleted stresstest --- stresstest.py | 234 -------------------------------------------------- 1 file changed, 234 deletions(-) delete mode 100644 stresstest.py diff --git a/stresstest.py b/stresstest.py deleted file mode 100644 index b1610ed..0000000 --- a/stresstest.py +++ /dev/null @@ -1,234 +0,0 @@ -import pandas as pd -import json -import pytz -from datetime import datetime, timezone, timedelta -import time -from src.factryengine.models import Resource, Task, Assignment, ResourceGroup -from src.factryengine.scheduler.core import Scheduler -from src.factryengine.scheduler.task_batch_processor import TaskSplitter - - -class ProdScheduler: - def __init__(self) -> None: - # Scheduler Attributes - self.cph_timezone = pytz.timezone('Europe/Copenhagen') - # self.today = datetime.now(timezone.utc).replace( - # hour=0, minute=0, second=0, microsecond=0) - self.today = datetime(2024, 10, 9, 0, 0, 0, 0, tzinfo=timezone.utc) - self.today_str = str(self.today)[:19] - - # Component Attributes - self.dict_resource = {} - self.dict_resourcegroups = {} - self.tasks_list = [] - self.task_dict = {} - self.pred_dict = {} - self.flow_map = {} - self.pred_exploded = {} - - # Misc Attributes - self.start_time = time.time() - - def convert_to_minutes(self, datetime_str, start_time_obj): - datetime_obj = datetime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S%z') - diff_minutes = (datetime_obj - start_time_obj).total_seconds()/60 - return int(diff_minutes) - - def adjust_capacity(self, start, end, capacity): - return (end - start) * capacity + start - - def organize_predecessors(self, task: Task): - try: - list_predecessors = self.pred_dict[task.id] - # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW - if task.batch_id: # Check if task is microbatched - # Look for each predecessors that exist in the flow map - for predecessor in list_predecessors: - pred_batch_id = f'{predecessor}-{task.batch_id}' - if pred_batch_id in self.flow_map and pred_batch_id in self.pred_dict: # Check if pred is part of flow - # Check if pred-parent connection is correct - if self.flow_map[task.id]['predecessor'] == self.flow_map[pred_batch_id]['parent']: - self.pred_dict[task.id] = [pred_batch_id] - - elif pred_batch_id not in self.pred_dict and predecessor not in self.task_dict: - parent_predecessor = [] - for pred in self.pred_dict[task.id]: - parent_predecessor.extend(self.pred_dict[pred]) - self.pred_dict[task.id] = parent_predecessor - # ============================================================== UNCOMMENT TO USE MICROBATCH FLOW - - # Remove task batch id - task.set_batch_id(None) - - # Check for predecessors to be exploded - for predecessor in list_predecessors: - if predecessor in self.pred_exploded: - self.pred_dict[task.id].remove( - predecessor) # Remove original value - self.pred_dict[task.id].extend( - self.pred_exploded[predecessor]) # Add exploded batches - - except Exception as e: - return - - def set_predecessors(self, task: Task): - if not task.id in self.pred_dict: # if task is not in pred_dict, then it has no predecessors - return - - for pred_id in self.pred_dict[task.id]: - if pred_id in self.task_dict: # ensure predecessor exists in task_dict - pred_task = self.task_dict[pred_id] - - # Avoid adding a predecessor multiple times - if pred_task not in task.predecessors: - # set predecessors for the predecessor first - self.set_predecessors(pred_task) - task.predecessors.append(pred_task) - - def create_resource_object(self, resource_list): - # Generate slot based on schedule selected in NocoDB - for row in resource_list: - periods_list = [] - for sched in row['availability']: - if not sched['is_absent']: - start = self.convert_to_minutes( - sched['start_datetime'], self.today) - end = self.convert_to_minutes( - sched['end_datetime'], self.today) - # ========= Uncomment to use capacity - capacity = sched['capacity_percent'] - # capacity = None - periods_list.append((int(start), int(self.adjust_capacity( - start, end, capacity)) if capacity else int(end))) - - resource_id = int(row['resource_id']) - self.dict_resource[resource_id] = Resource( - id=resource_id, available_windows=periods_list) - - def create_resource_groups(self, resource_group_list): - # Generate Resource Groups - for x in resource_group_list: - resource_list = [] - resources = x['resource_id'] - for r in resources: - if r in self.dict_resource: - resource_list.append(self.dict_resource[r]) - - if resource_list: - self.dict_resourcegroups[x['resource_group_id']] = ResourceGroup( - id=x['resource_group_id'], resources=resource_list) - - def create_batch(self, task: Task, batch_size: int): - batches = TaskSplitter(task, batch_size).split_into_batches() - counter = 1 - for batch in batches: - batch.id = f"{task.id}-{counter}" - counter += 1 - - return batches - - def create_task_object(self, task_list): - for i in task_list: - rg_list = [] - task_id = i['taskno'] - duration = int(i['duration']) - priority = int(i['priority']) - quantity = int(i['quantity']) - # micro_batch_size = int( - # i['micro_batch_size']) if i['micro_batch_size'] else None - micro_batch_size = None - resource_group_id = i['resource_group_id'] - rg_list = [self.dict_resourcegroups[g] for g in resource_group_id if g in self.dict_resourcegroups] - predecessors = i['predecessors'] - parent_collection = i['parent_item_collection_id'] if micro_batch_size else None - predecessor_collection = i['predecessor_item_collection_id'] if micro_batch_size else None - - assignments = [] - # Create assignments - for x in resource_group_id: - if x in self.dict_resourcegroups: - assignments.append(Assignment(resource_groups= [self.dict_resourcegroups[x]], resource_count= 1)) - - # Temporarily add into component dicts - temp_task = Task(id=task_id, - duration=duration, - priority=priority, - assignments= assignments, - quantity=quantity) - - # Check for micro-batches - if not micro_batch_size: - self.task_dict[task_id] = temp_task # Add task to dictionary - - # Add predecessor to dictionary - self.pred_dict[task_id] = predecessors - else: - self.pred_dict[task_id] = predecessors - batches = self.create_batch(temp_task, micro_batch_size) - self.task_dict.update({task.id: task for task in batches}) - - # Temporarily copy the original predecessors for the new batches - self.pred_dict.update( - {task.id: predecessors for task in batches}) - self.flow_map.update({task.id: { - "parent": parent_collection, - "predecessor": predecessor_collection} for task in batches}) - self.pred_exploded[task_id] = [task.id for task in batches] - - # Organize predecessors for batches - for task in self.task_dict.values(): - self.organize_predecessors(task) - - # Add predecessors - for task in self.task_dict.values(): - self.task_dict[task.id].predecessor_ids = [x for x in self.pred_dict[task.id] if x in self.task_dict] # Predecessor needs to be existing in task dictionary - - # Build final task list - self.tasks_list = [value for key, - value in sorted(self.task_dict.items())] - - # Convert periods to time - def int_to_datetime(self, num, start_time): - try: - # Parse the start time string into a datetime object - start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") - - # Add the number of minutes to the start datetime - delta = timedelta(minutes=num) - result_datetime = start_datetime + delta - return result_datetime - - except Exception as e: - print(num) - - -# Open and read the JSON file -with open('inputs/resource.json', 'r') as file: - r_data = json.load(file) - -# Open and read the JSON file -with open('inputs/tasks_all.json', 'r') as file: - t_data = json.load(file) - - -# Open and read the JSON file -with open('inputs/resourcegroups.json', 'r') as file: - rg_data = json.load(file) - - - -prodscheduler = ProdScheduler() -prodscheduler.create_resource_object(r_data) -print("Resource Objects Created.") - -prodscheduler.create_resource_groups(rg_data) -print("Resource Groups Created.") - -prodscheduler.create_task_object(t_data) -print("Task Objects Created.") -print(f"Original Task Length: {len(t_data)} | Post-Batched Length: {len(prodscheduler.task_dict.values())}") - - -solution = Scheduler(prodscheduler.tasks_list, list(prodscheduler.dict_resource.values())).schedule() -print("Solution Created.") - From cb018dc52b622e8685a85077edc427c5e8459418 Mon Sep 17 00:00:00 2001 From: MJ Ducut Date: Wed, 23 Oct 2024 22:15:05 +0800 Subject: [PATCH 27/27] Added find_first_index function and a window counter --- .../heuristic_solver/task_allocator.py | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/factryengine/scheduler/heuristic_solver/task_allocator.py b/src/factryengine/scheduler/heuristic_solver/task_allocator.py index c2afa4c..b7084f5 100644 --- a/src/factryengine/scheduler/heuristic_solver/task_allocator.py +++ b/src/factryengine/scheduler/heuristic_solver/task_allocator.py @@ -205,13 +205,8 @@ def _get_resource_intervals( resource_windows_output[resource_id] = intervals return resource_windows_output - - - def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: - """ - Finds relevant indexes in the resource intervals where the resource is used. - """ - indexes = [] + + def _find_first_index(self, resource_intervals: np.ma.MaskedArray) -> int | None: # Shift the mask by 1 to align with the 'next' element comparison current_mask = resource_intervals.mask[:-1] next_mask = resource_intervals.mask[1:] @@ -224,9 +219,36 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: indices = np.where(condition)[0] first_index = indices[0] if len(indices) > 0 else 0 + + next_non_zero_index = np.where( + (~resource_intervals.mask[first_index + 2:]) # Non-masked (non-zero) + & (resource_intervals.mask[first_index + 1:-1]) # Previous value masked + )[0] + + # Adjust x to align with the original array's indices + next_non_zero_index = ( + (first_index + 2 + next_non_zero_index[0]) if len(next_non_zero_index) > 0 else None + ) + + if next_non_zero_index and resource_intervals[next_non_zero_index] == resource_intervals[first_index+1]: + first_index = next_non_zero_index + + return first_index + + + def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: + """ + Finds relevant indexes in the resource intervals where the resource is used. + """ + # Mask where the data in resource_intervals is 0 + resource_intervals = np.ma.masked_where(resource_intervals == 0.0, resource_intervals) + + indexes = [] + first_index = self._find_first_index(resource_intervals) last_index = resource_intervals.size-1 indexes = [first_index] # Start with the first index + is_last_window_start = True # Flag to indicate the start of a window # Iterate through the range between first and last index for i in range(first_index, last_index + 1): @@ -248,23 +270,27 @@ def _find_indexes(self, resource_intervals: np.ma.MaskedArray) -> int | None: continue # Detect end of a window - if current > 0 and current == next_value and (is_prev_masked or previous < current): + if current > 0 and current == next_value and (is_prev_masked or previous < current) and is_last_window_start: indexes.append(i) + is_last_window_start = False continue # Detect end of window using masks - if current > 0 and is_next_masked and not is_curr_masked: + if current > 0 and is_next_masked and not is_curr_masked and is_last_window_start: indexes.append(i) + is_last_window_start = False continue # Detect start of a new window - if current > 0 and next_value > current and (is_prev_masked or previous == current): + if current > 0 and next_value > current and (is_prev_masked or previous == current) and not is_last_window_start: indexes.append(i) + is_last_window_start = True continue # Detect start of window using masks - if is_curr_masked and previous > 0 and next_value > 0: + if is_curr_masked and previous > 0 and next_value > 0 and not is_last_window_start: indexes.append(i) + is_last_window_start = True continue # Always add the last index