-
Notifications
You must be signed in to change notification settings - Fork 242
/
bryan_baseline_v3.py
141 lines (130 loc) · 5.57 KB
/
bryan_baseline_v3.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
# coding=utf-8
# @author:bryan
# blog: https://blog.csdn.net/bryan__
# github: https://github.com/YouChouNoBB/2018-tencent-ad-competition-baseline
import pandas as pd
import lightgbm as lgb
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import OneHotEncoder,LabelEncoder
from scipy import sparse
import os
import gc
import math
import numpy as np
def get_user_feature():
if os.path.exists('../data/userFeature.csv'):
user_feature=pd.read_csv('../data/userFeature.csv')
else:
userFeature_data = []
with open('../data/userFeature.data', 'r') as f:
cnt = 0
for i, line in enumerate(f):
line = line.strip().split('|')
userFeature_dict = {}
for each in line:
each_list = each.split(' ')
userFeature_dict[each_list[0]] = ' '.join(each_list[1:])
userFeature_data.append(userFeature_dict)
if i % 100000 == 0:
print(i)
if i % 1000000 == 0:
user_feature = pd.DataFrame(userFeature_data)
user_feature.to_csv('../data/userFeature_' + str(cnt) + '.csv', index=False)
cnt += 1
del userFeature_data, user_feature
userFeature_data = []
user_feature = pd.DataFrame(userFeature_data)
user_feature.to_csv('../data/userFeature_' + str(cnt) + '.csv', index=False)
del userFeature_data, user_feature
user_feature = pd.concat(
[pd.read_csv('../data/userFeature_' + str(i) + '.csv') for i in range(cnt + 1)]).reset_index(drop=True)
user_feature.to_csv('../data/userFeature.csv', index=False)
return user_feature
def get_data():
if os.path.exists('../data/data.csv'):
return pd.read_csv('../data/data.csv')
else:
ad_feature = pd.read_csv('../data/adFeature.csv')
train=pd.read_csv('../data/train.csv')
predict=pd.read_csv('../data/test1.csv')
train.loc[train['label']==-1,'label']=0
predict['label']=-1
user_feature=get_user_feature()
data=pd.concat([train,predict])
data=pd.merge(data,ad_feature,on='aid',how='left')
data=pd.merge(data,user_feature,on='uid',how='left')
data=data.fillna('-1')
del user_feature
return data
def batch_predict(data,index):
one_hot_feature=['LBS','age','carrier','consumptionAbility','education','gender','house','os','ct','marriageStatus','advertiserId','campaignId', 'creativeId',
'adCategoryId', 'productId', 'productType']
vector_feature=['appIdAction','appIdInstall','interest1','interest2','interest3','interest4','interest5','kw1','kw2','kw3','topic1','topic2','topic3']
for feature in one_hot_feature:
try:
data[feature] = LabelEncoder().fit_transform(data[feature].apply(int))
except:
data[feature] = LabelEncoder().fit_transform(data[feature])
train=data[data.label!=-1]
train_y=train.pop('label')
test=data[data.label==-1]
res=test[['aid','uid']]
test=test.drop('label',axis=1)
enc = OneHotEncoder()
train_x=train[['creativeSize']]
test_x=test[['creativeSize']]
for feature in one_hot_feature:
enc.fit(data[feature].values.reshape(-1, 1))
train_a=enc.transform(train[feature].values.reshape(-1, 1))
test_a = enc.transform(test[feature].values.reshape(-1, 1))
train_x= sparse.hstack((train_x, train_a))
test_x = sparse.hstack((test_x, test_a))
print(feature+' finish')
print('one-hot prepared !')
cv=CountVectorizer()
for feature in vector_feature:
cv.fit(data[feature])
train_a = cv.transform(train[feature])
test_a = cv.transform(test[feature])
train_x = sparse.hstack((train_x, train_a))
test_x = sparse.hstack((test_x, test_a))
print(feature + ' finish')
print('cv prepared !')
del data
return LGB_predict(train_x, train_y, test_x, res,index)
def LGB_predict(train_x,train_y,test_x,res,index):
print("LGB test")
clf = lgb.LGBMClassifier(
boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,
max_depth=-1, n_estimators=1500, objective='binary',
subsample=0.7, colsample_bytree=0.7, subsample_freq=1,
learning_rate=0.05, min_child_weight=50, random_state=2018, n_jobs=-1
)
clf.fit(train_x, train_y, eval_set=[(train_x, train_y)], eval_metric='auc',early_stopping_rounds=100)
res['score'+str(index)] = clf.predict_proba(test_x)[:,1]
res['score'+str(index)] = res['score'+str(index)].apply(lambda x: float('%.6f' % x))
print(str(index)+' predict finish!')
gc.collect()
res=res.reset_index(drop=True)
return res['score'+str(index)]
#数据分片处理，对每片分别训练预测，然后求平均
data=get_data()
train=data[data['label']!=-1]
test=data[data['label']==-1]
del data
predict=pd.read_csv('../data/test1.csv')
cnt=20
size = math.ceil(len(train) / cnt)
result=[]
for i in range(cnt):
start = size * i
end = (i + 1) * size if (i + 1) * size < len(train) else len(train)
slice = train[start:end]
result.append(batch_predict(pd.concat([slice,test]),i))
gc.collect()
result=pd.concat(result,axis=1)
result['score']=np.mean(result,axis=1)
result=result.reset_index(drop=True)
result=pd.concat([predict[['aid','uid']].reset_index(drop=True),result['score']],axis=1)
result[['aid','uid','score']].to_csv('../data/submission.csv', index=False)
os.system('zip ../data/baseline.zip ../data/submission.csv')