add experiments folder

experiments/README.md

@@ -0,0 +1,5 @@
+# course-allocation-experiment
+Experiments in fair course allocation.
+To run the experiments, you have to install, in addition to fairpyx:
+
+    pip install experiments-csv[plotting]

experiments/compare_course_allocation_algorithms.py

@@ -0,0 +1,172 @@
+"""
+Compare the performance of algorithms for fair course allocation.
+
+Programmer: Erel Segal-Halevi
+Since: 2023-07
+"""
+
+
+######### COMMON VARIABLES AND ROUTINES ##########
+
+from fairpyx import divide, AgentBundleValueMatrix, Instance
+import fairpyx.algorithms as crs
+from typing import *
+import numpy as np
+
+max_value = 1000
+normalized_sum_of_values = 1000
+TIME_LIMIT = 100
+
+algorithms_to_check = [
+    crs.utilitarian_matching, 
+    crs.iterated_maximum_matching_unadjusted, 
+    crs.iterated_maximum_matching_adjusted, 
+    crs.serial_dictatorship,                  # Very bad performance
+    crs.round_robin, 
+    crs.bidirectional_round_robin,
+    crs.almost_egalitarian_without_donation,
+    crs.almost_egalitarian_with_donation,
+    ]
+
+def evaluate_algorithm_on_instance(algorithm, instance):
+    allocation = divide(algorithm, instance)
+    matrix = AgentBundleValueMatrix(instance, allocation)
+    matrix.use_normalized_values()
+    return {
+        "utilitarian_value": matrix.utilitarian_value(),
+        "egalitarian_value": matrix.egalitarian_value(),
+        "max_envy": matrix.max_envy(),
+        "mean_envy": matrix.mean_envy(),
+        "max_deficit": matrix.max_deficit(),
+        "mean_deficit": matrix.mean_deficit(),
+        "num_with_top_1": matrix.count_agents_with_top_rank(1),
+        "num_with_top_2": matrix.count_agents_with_top_rank(2),
+        "num_with_top_3": matrix.count_agents_with_top_rank(3),
+    }
+
+
+
+######### EXPERIMENT WITH UNIFORMLY-RANDOM DATA ##########
+
+def course_allocation_with_random_instance_uniform(
+    num_of_agents:int, num_of_items:int, 
+    value_noise_ratio:float,
+    algorithm:Callable,
+    random_seed: int,):
+    agent_capacity_bounds =  [6,6]
+    item_capacity_bounds = [40,40]    
+    np.random.seed(random_seed)
+    instance = Instance.random_uniform(
+        num_of_agents=num_of_agents, num_of_items=num_of_items, 
+        normalized_sum_of_values=normalized_sum_of_values,
+        agent_capacity_bounds=agent_capacity_bounds, 
+        item_capacity_bounds=item_capacity_bounds, 
+        item_base_value_bounds=[1,max_value],
+        item_subjective_ratio_bounds=[1-value_noise_ratio, 1+value_noise_ratio]
+        )
+    return evaluate_algorithm_on_instance(algorithm, instance)
+
+def run_uniform_experiment():
+    # Run on uniformly-random data:
+    experiment = experiments_csv.Experiment("results/", "course_allocation_uniform.csv", backup_folder="results/backup/")
+    input_ranges = {
+        "num_of_agents": [100,200,300],         
+        "num_of_items":  [25],                   
+        "value_noise_ratio": [0, 0.2, 0.5, 0.8, 1],                        
+        "algorithm": algorithms_to_check,
+        "random_seed": range(5),
+    }
+    experiment.run_with_time_limit(course_allocation_with_random_instance_uniform, input_ranges, time_limit=TIME_LIMIT)
+
+
+
+######### EXPERIMENT WITH DATA GENERATED ACCORDING TO THE SZWS MODEL ##########
+
+def course_allocation_with_random_instance_szws(
+    num_of_agents:int, num_of_items:int, 
+    agent_capacity:int,
+    supply_ratio:float,
+    num_of_popular_items:int,
+    mean_num_of_favorite_items:float,
+    favorite_item_value_bounds:tuple[int,int],
+    nonfavorite_item_value_bounds:tuple[int,int],
+    algorithm:Callable,
+    random_seed: int,):
+    np.random.seed(random_seed)
+    instance = Instance.random_szws(
+        num_of_agents=num_of_agents, num_of_items=num_of_items, normalized_sum_of_values=normalized_sum_of_values,
+        agent_capacity=agent_capacity, 
+        supply_ratio=supply_ratio, 
+        num_of_popular_items=num_of_popular_items,
+        mean_num_of_favorite_items=mean_num_of_favorite_items,
+        favorite_item_value_bounds=favorite_item_value_bounds,
+        nonfavorite_item_value_bounds=nonfavorite_item_value_bounds,
+        )
+    return evaluate_algorithm_on_instance(algorithm, instance)
+
+def run_szws_experiment():
+    # Run on SZWS simulated data:
+    experiment = experiments_csv.Experiment("results/", "course_allocation_szws.csv", backup_folder="results/backup/")
+    input_ranges = {
+        "num_of_agents": [100,200,300],         
+        "num_of_items":  [25],                            # in SZWS: 25
+        "agent_capacity": [5],                            # as in SZWS 
+        "supply_ratio": [1.1, 1.25, 1.5],                    # as in SZWS
+        "num_of_popular_items": [6, 9],                   # as in SZWS
+        "mean_num_of_favorite_items": [2.6, 3.85],        # as in SZWS code https://github.com/marketdesignresearch/Course-Match-Preference-Simulator/blob/main/preference_generator_demo.ipynb
+        "favorite_item_value_bounds": [(50,100)],         # as in SZWS code https://github.com/marketdesignresearch/Course-Match-Preference-Simulator/blob/main/preference_generator.py
+        "nonfavorite_item_value_bounds": [(0,50)],        # as in SZWS code https://github.com/marketdesignresearch/Course-Match-Preference-Simulator/blob/main/preference_generator.py
+        "algorithm": algorithms_to_check,
+        "random_seed": range(5),
+    }
+    experiment.run_with_time_limit(course_allocation_with_random_instance_szws, input_ranges, time_limit=TIME_LIMIT)
+
+
+
+######### EXPERIMENT WITH DATA SAMPLED FROM ARIEL 5783 DATA ##########
+
+import json
+filename = "data/ariel_5783_input.json"
+with open(filename, "r", encoding="utf-8") as file:
+    ariel_5783_input = json.load(file)
+
+def course_allocation_with_random_instance_sample(
+    max_total_agent_capacity:int, 
+    algorithm:Callable,
+    random_seed: int,):
+    np.random.seed(random_seed)
+
+    (valuations, agent_capacities, item_capacities, agent_conflicts, item_conflicts) = \
+        (ariel_5783_input["valuations"], ariel_5783_input["agent_capacities"], ariel_5783_input["item_capacities"], ariel_5783_input["agent_conflicts"], ariel_5783_input["item_conflicts"])
+    instance = Instance.random_sample(
+        max_num_of_agents = max_total_agent_capacity, 
+        max_total_agent_capacity = max_total_agent_capacity,
+        prototype_agent_conflicts=agent_conflicts,
+        prototype_agent_capacities=agent_capacities, 
+        prototype_valuations=valuations,
+        item_capacities=item_capacities,
+        item_conflicts=item_conflicts)
+    return evaluate_algorithm_on_instance(algorithm, instance)
+
+def run_ariel_experiment():
+    # Run on Ariel sample data:
+    experiment = experiments_csv.Experiment("results/", "course_allocation_ariel.csv", backup_folder="results/backup/")
+    input_ranges = {
+        "max_total_agent_capacity": [1000, 1115, 1500, 2000], # in reality: 1115
+        "algorithm": algorithms_to_check,
+        "random_seed": range(10),
+    }
+    experiment.run_with_time_limit(course_allocation_with_random_instance_sample, input_ranges, time_limit=TIME_LIMIT)
+
+
+
+######### MAIN PROGRAM ##########
+
+if __name__ == "__main__":
+    import logging, experiments_csv
+    experiments_csv.logger.setLevel(logging.INFO)
+    run_uniform_experiment()
+    run_szws_experiment()
+    run_ariel_experiment()
+
+

experiments/data/README.md

@@ -0,0 +1,3 @@
+Some anonymized data files for experiments on course allocation.
+
+* [ariel_5783_input.json](ariel_5783_input.json): data from prototype course-allocation website in Ariel University, Computer Science Department, year 5783.

experiments/plot_simulation_results.py

@@ -0,0 +1,105 @@
+from experiments_csv import single_plot_results, multi_plot_results
+from matplotlib import pyplot as plt
+from pathlib import Path
+import sys
+
+def multi_multi_plot_results(results_csv_file:str, save_to_file_template:str, filter:dict, 
+     x_field:str, y_fields:list[str], z_field:str, mean:bool, 
+     subplot_field:str, subplot_rows:int, subplot_cols:int, sharey:bool, sharex:bool,
+     legend_properties:dict):
+     for y_field in y_fields:
+          save_to_file=save_to_file_template.format(y_field)
+          print(y_field, save_to_file)
+          multi_plot_results(
+               results_csv_file=results_csv_file,
+               save_to_file=save_to_file,
+               filter=filter, 
+               x_field=x_field, y_field=y_field, z_field=z_field, mean=mean, 
+               subplot_field=subplot_field, subplot_rows=subplot_rows, subplot_cols=subplot_cols, sharey=sharey, sharex=sharex,
+               legend_properties=legend_properties,
+               )
+
+
+def plot_course_allocation_results_szws():
+     filter={"num_of_agents": 100, "num_of_items": 25}
+     y_fields=["utilitarian_value","egalitarian_value", "max_envy", "mean_envy",  "mean_deficit", "max_deficit", "num_with_top_1", "num_with_top_2", "num_with_top_3","runtime"]
+     multi_multi_plot_results(
+          results_csv_file="results/course_allocation_szws.csv", 
+          save_to_file_template="results/course_allocation_szws_{}.png",
+          filter=filter, 
+          x_field="supply_ratio", y_fields=y_fields, z_field="algorithm", mean=True,
+          subplot_field="num_of_popular_items", subplot_rows=2, subplot_cols=1, sharey=True, sharex=True,
+          legend_properties={"size":6}, 
+          )
+
+
+def plot_course_allocation_results_ariel():
+     y_fields=["utilitarian_value","egalitarian_value", "max_envy", "mean_envy",  "mean_deficit", "max_deficit", "num_with_top_1", "num_with_top_2", "num_with_top_3","runtime"]
+     multi_multi_plot_results(
+          results_csv_file="results/course_allocation_szws.csv", 
+          save_to_file_template="results/course_allocation_szws_{}.png",
+          filter=filter, 
+          x_field="supply_ratio", y_fields=y_fields, z_field="algorithm", mean=True,
+          subplot_field="num_of_popular_items", subplot_rows=2, subplot_cols=1, sharey=True, sharex=True,
+          legend_properties={"size":6}, 
+          )
+
+
+
+
+def plot_course_allocation_results_uniform():
+     filter={"num_of_items": 20, 
+          "algorithm": [
+               "yekta_day", 
+               "iterated_maximum_matching_unadjusted","iterated_maximum_matching_adjusted",
+               "almost_egalitarian_without_donation","almost_egalitarian_with_donation",
+               "round_robin", "bidirectional_round_robin"
+               ]}
+     y_fields=["utilitarian_value","egalitarian_value", "max_envy", "mean_envy",  "mean_deficit", "max_deficit", "num_with_top_1", "num_with_top_2", "num_with_top_3","runtime"]
+     multi_multi_plot_results(
+          results_csv_file="results/course_allocation_uniform.csv", 
+          save_to_file_template="results/course_allocation_uniform_{}.png",
+          filter=filter, 
+          x_field="value_noise_ratio", y_fields=y_fields, z_field="algorithm", mean=True,
+          subplot_field = "num_of_agents", subplot_rows=2, subplot_cols=2, sharey=True, sharex=True,
+          legend_properties={"size":6}, 
+          )
+
+
+# plot_course_allocation_results_uniform()
+plot_course_allocation_results_szws()
+
+
+
+######## OLD PLOTS
+
+
+
+
+# multi_plot_results(
+#      "results/fractional_course_allocation.csv", 
+#      save_to_file=True,
+#     #  filter={"num_of_items": [5,10,20,30]},  # ValueError: ('Lengths must match to compare', (760,), (4,))
+#      filter={}, 
+#      x_field="num_of_agents", y_field="runtime", z_field="algorithm", mean=True, 
+#      subplot_field = "num_of_items", subplot_rows=2, subplot_cols=2, sharey=True, sharex=True,
+#      legend_properties={"size":6}, 
+#      )
+
+
+# multi_plot_results(
+#      "results/check_effect_of_name_size.csv", 
+#      save_to_file="results/check_effect_of_name_size.png",
+#      filter={}, 
+#      x_field="agent_name_size", y_field="runtime", z_field="algorithm", mean=True, 
+#      subplot_field = "item_name_size", subplot_rows=2, subplot_cols=2, sharey=True, sharex=True,
+#      legend_properties={"size":6}, 
+#      )
+
+# multi_plot_results(
+#      "results/many_to_many_matchings.csv", save_to_file=True,
+#      filter={}, 
+#      x_field="item_capacity", y_field="runtime", z_field="algorithm", subplot_field = "agent_capacity", 
+#      mean=True, subplot_rows=2, subplot_cols=3, sharey=True, sharex=True,
+#      legend_properties={"size":6}, ylim=(0,30), xlim=(0,40))
+