forked from FederatedAI/FATE
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dataio_param.py
160 lines (124 loc) · 8.09 KB
/
dataio_param.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2019 The FATE Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from pipeline.param.base_param import BaseParam
class DataIOParam(BaseParam):
"""
Define dataio parameters that used in federated ml.
This module is not supported to use in training task since Fate-v1.9.0, use data transform instead.
Parameters
----------
input_format : str, accepted 'dense','sparse' 'tag' only in this version. default: 'dense'.
please have a look at this tutorial at "DataIO" section of federatedml/util/README.md.
Formally,
dense input format data should be set to "dense",
svm-light input format data should be set to "sparse",
tag or tag:value input format data should be set to "tag".
delimitor : str, the delimitor of data input, default: ','
data_type : str, the data type of data input, accepted 'float','float64','int','int64','str','long'
"default: "float64"
exclusive_data_type : dict, the key of dict is col_name, the value is data_type, use to specified special data type
of some features.
tag_with_value: bool, use if input_format is 'tag', if tag_with_value is True,
input column data format should be tag[delimitor]value, otherwise is tag only
tag_value_delimitor: str, use if input_format is 'tag' and 'tag_with_value' is True,
delimitor of tag[delimitor]value column value.
missing_fill : bool, need to fill missing value or not, accepted only True/False, default: True
default_value : None or single object type or list, the value to replace missing value.
if None, it will use default value define in federatedml/feature/imputer.py,
if single object, will fill missing value with this object,
if list, it's length should be the sample of input data' feature dimension,
means that if some column happens to have missing values, it will replace it
the value by element in the identical position of this list.
default: None
missing_fill_method: None or str, the method to replace missing value, should be one of [None, 'min', 'max', 'mean', 'designated'], default: None
missing_impute: None or list, element of list can be any type, or auto generated if value is None, define which values to be consider as missing, default: None
outlier_replace: bool, need to replace outlier value or not, accepted only True/False, default: True
outlier_replace_method: None or str, the method to replace missing value, should be one of [None, 'min', 'max', 'mean', 'designated'], default: None
outlier_impute: None or list, element of list can be any type, which values should be regard as missing value, default: None
outlier_replace_value: None or single object type or list, the value to replace outlier.
if None, it will use default value define in federatedml/feature/imputer.py,
if single object, will replace outlier with this object,
if list, it's length should be the sample of input data' feature dimension,
means that if some column happens to have outliers, it will replace it
the value by element in the identical position of this list.
default: None
with_label : bool, True if input data consist of label, False otherwise. default: 'false'
label_name : str, column_name of the column where label locates, only use in dense-inputformat. default: 'y'
label_type : object, accepted 'int','int64','float','float64','long','str' only,
use when with_label is True. default: 'false'
output_format : str, accepted 'dense','sparse' only in this version. default: 'dense'
"""
def __init__(self, input_format="dense", delimitor=',', data_type='float64',
exclusive_data_type=None,
tag_with_value=False, tag_value_delimitor=":",
missing_fill=False, default_value=0, missing_fill_method=None,
missing_impute=None, outlier_replace=False, outlier_replace_method=None,
outlier_impute=None, outlier_replace_value=0,
with_label=False, label_name='y',
label_type='int', output_format='dense', need_run=True):
self.input_format = input_format
self.delimitor = delimitor
self.data_type = data_type
self.exclusive_data_type = exclusive_data_type
self.tag_with_value = tag_with_value
self.tag_value_delimitor = tag_value_delimitor
self.missing_fill = missing_fill
self.default_value = default_value
self.missing_fill_method = missing_fill_method
self.missing_impute = missing_impute
self.outlier_replace = outlier_replace
self.outlier_replace_method = outlier_replace_method
self.outlier_impute = outlier_impute
self.outlier_replace_value = outlier_replace_value
self.with_label = with_label
self.label_name = label_name
self.label_type = label_type
self.output_format = output_format
self.need_run = need_run
def check(self):
descr = "dataio param's"
self.input_format = self.check_and_change_lower(self.input_format,
["dense", "sparse", "tag"],
descr)
self.output_format = self.check_and_change_lower(self.output_format,
["dense", "sparse"],
descr)
self.data_type = self.check_and_change_lower(self.data_type,
["int", "int64", "float", "float64", "str", "long"],
descr)
if type(self.missing_fill).__name__ != 'bool':
raise ValueError("dataio param's missing_fill {} not supported".format(self.missing_fill))
if self.missing_fill_method is not None:
self.missing_fill_method = self.check_and_change_lower(self.missing_fill_method,
['min', 'max', 'mean', 'designated'],
descr)
if self.outlier_replace_method is not None:
self.outlier_replace_method = self.check_and_change_lower(self.outlier_replace_method,
['min', 'max', 'mean', 'designated'],
descr)
if type(self.with_label).__name__ != 'bool':
raise ValueError("dataio param's with_label {} not supported".format(self.with_label))
if self.with_label:
if not isinstance(self.label_name, str):
raise ValueError("dataio param's label_name {} should be str".format(self.label_name))
self.label_type = self.check_and_change_lower(self.label_type,
["int", "int64", "float", "float64", "str", "long"],
descr)
if self.exclusive_data_type is not None and not isinstance(self.exclusive_data_type, dict):
raise ValueError("exclusive_data_type is should be None or a dict")
return True