forked from FG-AI4H/health-aiaudit-benchmark-starters
-
Notifications
You must be signed in to change notification settings - Fork 0
/
challenge_config.yaml
executable file
·433 lines (429 loc) · 12.9 KB
/
challenge_config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
# If you are not sure what all these fields mean, please refer our documentation here:
# https://evalai.readthedocs.io/en/latest/configuration.html
title: Random Number Generator Challenge
short_description: Random number generation challenge for each submission
description: templates/description.html
evaluation_details: templates/evaluation_details.html
terms_and_conditions: templates/terms_and_conditions.html
image: logo.jpg
submission_guidelines: templates/submission_guidelines.html
leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis.
evaluation_script: evaluation_script.zip
remote_evaluation: False
is_docker_based: False
start_date: 2019-01-01 00:00:00
end_date: 2099-05-31 23:59:59
published: True
leaderboard:
- id: 1
schema:
{
"labels": ["Metric1", "Metric2", "Metric3", "Total"],
"default_order_by": "Total",
"metadata": {
"Metric1": {
"sort_ascending": True,
"description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
},
"Metric2": {
"sort_ascending": True,
"description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
}
}
}
challenge_phases:
- id: 1
name: Dev Phase
description: templates/challenge_phase_1_description.html
leaderboard_public: False
is_public: True
is_submission_public: True
start_date: 2019-01-19 00:00:00
end_date: 2099-04-25 23:59:59
test_annotation_file: annotations/test_annotations_devsplit.json
codename: dev
max_submissions_per_day: 5
max_submissions_per_month: 50
max_submissions: 50
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
- name: pl_reg_int_use_prd_name
is_visible: True
- name: pl_reg_int_use_prd_cl_use
is_visible: True
- name: pl_reg_int_use_prd_tsk
is_visible: True
- name: pl_reg_int_use_prd_cat
is_visible: True
- name: pl_reg_int_use_prd_usrgrp
is_visible: True
- name: pl_reg_int_use_prd_opmod
is_visible: True
- name: pl_reg_int_use_prd_autl
is_visible: True
- name: dc_tv_pl_dat_src
is_visible: True
- name: dc_tv_pl_dat_numsamp
is_visible: True
- name: dc_tv_pl_dat_mod
is_visible: True
- name: dc_tv_pl_dat_noise
is_visible: True
- name: dc_tv_pl_dat_mis
is_visible: True
- name: dc_tv_pl_dat_inc_exc
is_visible: True
- name: dc_reg_dat_priv
is_visible: True
- name: dc_eth_dat_priv_cons
is_visible: True
- name: dc_eth_dat_priv_anon
is_visible: True
- name: dp_tv_numsamp
is_visible: True
- name: dp_tv_gndtrh
is_visible: True
- name: dp_tv_lbl
is_visible: True
- name: dp_tv_part
is_visible: True
- name: dp_tv_ovp
is_visible: True
- name: dp_eth_de_id
is_visible: True
- name: dp_reg_gndt
is_visible: True
- name: dp_tv_prev
is_visible: True
- name: dp_tv_preprop
is_visible: True
- name: mt_eth_sen
is_visible: True
- name: mt_tv_arch
is_visible: True
- name: mtun_tv_sen
is_visible: True
- name: mtun_tv_val
is_visible: True
- name: mtun_tv_fet
is_visible: True
- name: mtun_cl_decth
is_visible: True
- name: mtun_cl_set
is_visible: True
- name: meval_eth_usr
is_visible: True
- name: meval_tv_opt
is_visible: True
- name: meval_tv_param
is_visible: True
- name: meval_tv_gen
is_visible: True
- name: meval_tv_per_comp
is_visible: True
- name: meval_tv_per_comp_dis
is_visible: True
- name: meval_tv_intp_tdoff
is_visible: True
- name: meval_cl_permet_crt
is_visible: True
- name: meval_cl_patsaf
is_visible: True
- name: meval_cl_mod_spec
is_visible: True
- name: meval_cl_rskprob
is_visible: True
- name: mdeploy_cl_lat
is_visible: True
- name: mdeploy_cl_prac
is_visible: True
- name: mdeploy_cl_care_qual
is_visible: True
- name: mdeploy_cl_wkfl_int
is_visible: True
- name: mdeploy_cl_rsk_prob
is_visible: True
- name: mdeploy_cl_inp_feat
is_visible: True
- name: mdeploy_cl_recom
is_visible: True
- name: mdeploy_cl_patcom
is_visible: True
- name: mdeploy_reg_saf
is_visible: True
- name: mmain_reg_sys
is_visible: True
- name: mmain_reg_contlr
is_visible: True
- name: pl_reg_int_use_pat_patgrp
is_visible: True
- name: pl_reg_int_use_md_devplr
is_visible: True
- name: pl_reg_int_use_md_devkt
is_visible: True
- name: pl_reg_int_use_md_devperiod
is_visible: True
- name: pl_reg_int_use_md_reg
is_visible: True
- name: pl_reg_int_use_md_lic
is_visible: True
- name: dc_tv_pl_dat_coll_plc
is_visible: True
- name: dc_tv_pl_dat_coll_period
is_visible: True
- name: dc_tv_pl_dat_coll_auth
is_visible: True
- name: dc_tv_pl_dat_coll_fund
is_visible: True
- name: dc_tv_pl_dat_samprate
is_visible: True
- name: dc_tv_pl_dat_dim
is_visible: True
- name: dc_tv_pl_dat_mod
is_visible: True
- name: dc_tv_pl_dat_devtyp
is_visible: True
- name: dc_eth_dat_incl_excl
is_visible: True
- name: dp_eth_saf_sec_prtcol
is_visible: True
- name: mt_tv_out
is_visible: True
- name: meval_tv_param_config
is_visible: True
submission_meta_attributes:
- name: TextAttribute
description: Sample
type: text
required: False
- name: SingleOptionAttribute
description: Sample
type: radio
options: ["A", "B", "C"]
- name: MultipleChoiceAttribute
description: Sample
type: checkbox
options: ["alpha", "beta", "gamma"]
- name: TrueFalseField
description: Sample
type: boolean
required: True
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz"
- id: 2
name: Test Phase
description: templates/challenge_phase_2_description.html
leaderboard_public: True
is_public: True
is_submission_public: True
start_date: 2019-01-01 00:00:00
end_date: 2099-05-24 23:59:59
test_annotation_file: annotations/test_annotations_testsplit.json
codename: test
max_submissions_per_day: 5
max_submissions_per_month: 50
max_submissions: 50
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
- name: pl_reg_int_use_prd_name
is_visible: True
- name: pl_reg_int_use_prd_cl_use
is_visible: True
- name: pl_reg_int_use_prd_tsk
is_visible: True
- name: pl_reg_int_use_prd_cat
is_visible: True
- name: pl_reg_int_use_prd_usrgrp
is_visible: True
- name: pl_reg_int_use_prd_opmod
is_visible: True
- name: pl_reg_int_use_prd_autl
is_visible: True
- name: dc_tv_pl_dat_src
is_visible: True
- name: dc_tv_pl_dat_numsamp
is_visible: True
- name: dc_tv_pl_dat_mod
is_visible: True
- name: dc_tv_pl_dat_noise
is_visible: True
- name: dc_tv_pl_dat_mis
is_visible: True
- name: dc_tv_pl_dat_inc_exc
is_visible: True
- name: dc_reg_dat_priv
is_visible: True
- name: dc_eth_dat_priv_cons
is_visible: True
- name: dc_eth_dat_priv_anon
is_visible: True
- name: dp_tv_numsamp
is_visible: True
- name: dp_tv_gndtrh
is_visible: True
- name: dp_tv_lbl
is_visible: True
- name: dp_tv_part
is_visible: True
- name: dp_tv_ovp
is_visible: True
- name: dp_eth_de_id
is_visible: True
- name: dp_reg_gndt
is_visible: True
- name: dp_tv_prev
is_visible: True
- name: dp_tv_preprop
is_visible: True
- name: mt_eth_sen
is_visible: True
- name: mt_tv_arch
is_visible: True
- name: mtun_tv_sen
is_visible: True
- name: mtun_tv_val
is_visible: True
- name: mtun_tv_fet
is_visible: True
- name: mtun_cl_decth
is_visible: True
- name: mtun_cl_set
is_visible: True
- name: meval_eth_usr
is_visible: True
- name: meval_tv_opt
is_visible: True
- name: meval_tv_param
is_visible: True
- name: meval_tv_gen
is_visible: True
- name: meval_tv_per_comp
is_visible: True
- name: meval_tv_per_comp_dis
is_visible: True
- name: meval_tv_intp_tdoff
is_visible: True
- name: meval_cl_permet_crt
is_visible: True
- name: meval_cl_patsaf
is_visible: True
- name: meval_cl_mod_spec
is_visible: True
- name: meval_cl_rskprob
is_visible: True
- name: mdeploy_cl_lat
is_visible: True
- name: mdeploy_cl_prac
is_visible: True
- name: mdeploy_cl_care_qual
is_visible: True
- name: mdeploy_cl_wkfl_int
is_visible: True
- name: mdeploy_cl_rsk_prob
is_visible: True
- name: mdeploy_cl_inp_feat
is_visible: True
- name: mdeploy_cl_recom
is_visible: True
- name: mdeploy_cl_patcom
is_visible: True
- name: mdeploy_reg_saf
is_visible: True
- name: mmain_reg_sys
is_visible: True
- name: mmain_reg_contlr
is_visible: True
- name: pl_reg_int_use_pat_patgrp
is_visible: True
- name: pl_reg_int_use_md_devplr
is_visible: True
- name: pl_reg_int_use_md_devkt
is_visible: True
- name: pl_reg_int_use_md_devperiod
is_visible: True
- name: pl_reg_int_use_md_reg
is_visible: True
- name: pl_reg_int_use_md_lic
is_visible: True
- name: dc_tv_pl_dat_coll_plc
is_visible: True
- name: dc_tv_pl_dat_coll_period
is_visible: True
- name: dc_tv_pl_dat_coll_auth
is_visible: True
- name: dc_tv_pl_dat_coll_fund
is_visible: True
- name: dc_tv_pl_dat_samprate
is_visible: True
- name: dc_tv_pl_dat_dim
is_visible: True
- name: dc_tv_pl_dat_mod
is_visible: True
- name: dc_tv_pl_dat_devtyp
is_visible: True
- name: dc_eth_dat_incl_excl
is_visible: True
- name: dp_eth_saf_sec_prtcol
is_visible: True
- name: mt_tv_out
is_visible: True
- name: meval_tv_param_config
is_visible: True
submission_meta_attributes:
- name: TextAttribute
description: Sample
type: text
- name: SingleOptionAttribute
description: Sample
type: radio
options: ["A", "B", "C"]
- name: MultipleChoiceAttribute
description: Sample
type: checkbox
options: ["alpha", "beta", "gamma"]
- name: TrueFalseField
description: Sample
type: boolean
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
dataset_splits:
- id: 1
name: Train Split
codename: train_split
- id: 2
name: Test Split
codename: test_split
challenge_phase_splits:
- challenge_phase_id: 1
leaderboard_id: 1
dataset_split_id: 1
visibility: 1
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 1
visibility: 3
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 2
visibility: 1
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True