-
Notifications
You must be signed in to change notification settings - Fork 3
/
CONFIG.py
95 lines (67 loc) · 3.52 KB
/
CONFIG.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
from datetime import datetime
from os.path import join
class CONFIG:
# PATH TO DATA-FARM PROJECT
PROJECT_PATH = "/absolute/path/to/DataFarm/project/" # <absolute_path_to_project>
# PATH TO INPUT DATA
GENERATED_JOB_INPUT_DATA_PATH = "/absolute/path/to/input/data/" # <absolute_path_to_input_data>
# FLINK HOME
FLINK_HOME = "/absolute/path/to/flink" # <path_to_flink>
# EXPERIMENT ID
EXPERIMENT_ID = "Experiment1"
# DATA ID
DATA_ID = "1GB"
# Job generator conf
N_JOBS = 10
N_VERSIONS = 3
JOB_SEED = -1 # -1 | 0
DATA_MANAGER = "TPCH" # "TPCH" | "IMDB"
#################################################################
# DATA GENERATORS HOME
ABSTRACT_PLAN_GENERATOR = join(PROJECT_PATH, "generator_labeler")
JOB_GENERATOR = join(PROJECT_PATH, "instantiator")
# Path to original exec plans
ORIG_EXEC_PLAN_FOLDER = join(PROJECT_PATH, "data/input_workload_exec_plan/") # <absolute_path_to_input_workload>
# Path of generated data destination
BASE_PATH = join(PROJECT_PATH, "data/") # <absolute_path_to_input_data>
EXPERIMENT_PATH = join(BASE_PATH, EXPERIMENT_ID+"/")
GENERATED_ABSTRACT_EXECUTION_PLAN_FOLDER = join(EXPERIMENT_PATH, "generated_abstract_exec_plans")
GENERATED_JOB_FOLDER = join(EXPERIMENT_PATH, "generated_jobs")
GENERATED_JOB_EXEC_PLAN_FOLDER = join(EXPERIMENT_PATH, "generated_jobs_exec_plans")
GENERATED_JOB_TASK_MANAGER_DETAILS = join(EXPERIMENT_PATH, "generated_jobs_task_manager_details")
print(GENERATED_JOB_TASK_MANAGER_DETAILS)
# SUBMIT CONF
# Path where to store the execution plans of the generated jobs after their execution
GENERATED_JOB_OUTPUT_PLAN_PATH = join(GENERATED_JOB_EXEC_PLAN_FOLDER, DATA_ID+"/")
GENERATED_JOB_TASK_MANAGER_DETAILS_OUTPUT_PATH = join(GENERATED_JOB_TASK_MANAGER_DETAILS, DATA_ID+"/")
# Job Submission config
LOCAL = "nolocal" # [local|nolocal]
LOCAL_HEAP = "8GB" # [8GB|none]
PARALLELISM = "1"
# FLINK_TASK_MANAGER_URL
FLINK_TASK_MANAGER_URL = "http://127.0.0.1:8081/"
BUILD_SBT = "build.sbt"
RUN = True
########################################################
## Config Label Forecaster #############################
########################################################
exec_timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
LABEL_COL = "Log_netRunTime"
SAMPLE_COL = "Log_sourceCardinalitySum"
FEATURE_COLS = ["t_0", "t_1", "t_2", "t_3", "t_4", "t_5", "t_6",
"joinOutCard_sum", "joinOutCard_mean", "joinOutCard_std", "joinOutCard_min", "joinOutCard_max",
"filterOutCard_mean", "filterOutCard_std", "filterOutCard_min", "filterOutCard_max",
"groupbyOutCard_mean", "groupbyOutCard_std", "groupbyOutCard_min", "groupbyOutCard_max",
"outCardinality_mean", "outCardinality_std", "outCardinality_min", "outCardinality_max",
"outCardinality_kurtosis", "outCardinality_skew",
"sourceOutCard_mean", "sourceOutCard_std", "sourceOutCard_min", "sourceOutCard_max", "sourceCardinalitySum",
"complexity_mean", "complexity_min", "complexity_max"
]
MAX_EARLY_STOP = 2
EARLY_STOP_TH = 0.1 # Float between 0 and 1
MAX_ITER = 5
INIT_JOBS = 10
RANDOM_INIT = False
RANDOM_SAMPLING = False
GENERATED_METADATA_PATH = join(GENERATED_JOB_FOLDER, "generated_jobs_info.json")
LABEL_FORECASTER_OUT = join(EXPERIMENT_PATH, "label_forecaster_" + exec_timestamp)