-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.html
2303 lines (1049 loc) · 139 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!doctype html>
<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
<!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
<link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />
<link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />
<link href="/css/main.css?v=5.1.0" rel="stylesheet" type="text/css" />
<meta name="keywords" content="Hexo, NexT" />
<link rel="alternate" href="/atom.xml" title="江峰的技术博客" type="application/atom+xml" />
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.0" />
<meta property="og:type" content="website">
<meta property="og:title" content="江峰的技术博客">
<meta property="og:url" content="http://JiangFeng.github.com/index.html">
<meta property="og:site_name" content="江峰的技术博客">
<meta property="og:locale" content="zh-Hans">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="江峰的技术博客">
<script type="text/javascript" id="hexo.configurations">
var NexT = window.NexT || {};
var CONFIG = {
root: '/',
scheme: 'Pisces',
sidebar: {"position":"left","display":"post","offset":12,"offset_float":0,"b2t":false,"scrollpercent":false},
fancybox: true,
motion: true,
duoshuo: {
userId: '0',
author: '博主'
},
algolia: {
applicationID: '',
apiKey: '',
indexName: '',
hits: {"per_page":10},
labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
}
};
</script>
<link rel="canonical" href="http://JiangFeng.github.com/"/>
<title> 江峰的技术博客 </title><!-- hexo-inject:begin --><!-- hexo-inject:end -->
</head>
<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">
<!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="container one-collumn sidebar-position-left
page-home
">
<div class="headband"></div>
<header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
<div class="header-inner"><div class="site-brand-wrapper">
<div class="site-meta ">
<div class="custom-logo-site-title">
<a href="/" class="brand" rel="start">
<span class="logo-line-before"><i></i></span>
<span class="site-title">江峰的技术博客</span>
<span class="logo-line-after"><i></i></span>
</a>
</div>
<p class="site-subtitle"></p>
</div>
<div class="site-nav-toggle">
<button>
<span class="btn-bar"></span>
<span class="btn-bar"></span>
<span class="btn-bar"></span>
</button>
</div>
</div>
<nav class="site-nav">
<ul id="menu" class="menu">
<li class="menu-item menu-item-home">
<a href="/" rel="section">
<i class="menu-item-icon fa fa-fw fa-home"></i> <br />
首页
</a>
</li>
<li class="menu-item menu-item-archives">
<a href="/archives" rel="section">
<i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
归档
</a>
</li>
<li class="menu-item menu-item-categories">
<a href="/categories" rel="section">
<i class="menu-item-icon fa fa-fw fa-th"></i> <br />
分类
</a>
</li>
<li class="menu-item menu-item-tags">
<a href="/tags" rel="section">
<i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
标签
</a>
</li>
</ul>
</nav>
</div>
</header>
<main id="main" class="main">
<div class="main-inner">
<div class="content-wrap">
<div id="content" class="content">
<section id="posts" class="posts-expand">
<article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
<link itemprop="mainEntityOfPage" href="http://JiangFeng.github.com/2018/12/21/tensorflow-keras-api-进行文本分类/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Jiang Feng">
<meta itemprop="description" content="">
<meta itemprop="image" content="/images/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="江峰的技术博客">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2018/12/21/tensorflow-keras-api-进行文本分类/" itemprop="url">
tensorflow keras api 进行文本分类
</a>
</h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建于" itemprop="dateCreated datePublished" datetime="2018-12-21T18:28:06+08:00">
2018-12-21
</time>
</span>
<span class="post-category" >
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-folder-o"></i>
</span>
<span class="post-meta-item-text">分类于</span>
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/deeplearning/" itemprop="url" rel="index">
<span itemprop="name">deeplearning</span>
</a>
</span>
，
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/deeplearning/tensorflow/" itemprop="url" rel="index">
<span itemprop="name">tensorflow</span>
</a>
</span>
</span>
<span id="/2018/12/21/tensorflow-keras-api-进行文本分类/" class="leancloud_visitors" data-flag-title="tensorflow keras api 进行文本分类">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读次数 </span>
<span class="leancloud-visitors-count"></span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
</div>
<div>
</div>
<div>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
<link itemprop="mainEntityOfPage" href="http://JiangFeng.github.com/2018/01/16/TF-IDF算法/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Jiang Feng">
<meta itemprop="description" content="">
<meta itemprop="image" content="/images/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="江峰的技术博客">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2018/01/16/TF-IDF算法/" itemprop="url">
TF-IDF算法
</a>
</h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建于" itemprop="dateCreated datePublished" datetime="2018-01-16T20:04:55+08:00">
2018-01-16
</time>
</span>
<span class="post-category" >
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-folder-o"></i>
</span>
<span class="post-meta-item-text">分类于</span>
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/NLP/" itemprop="url" rel="index">
<span itemprop="name">NLP</span>
</a>
</span>
</span>
<span id="/2018/01/16/TF-IDF算法/" class="leancloud_visitors" data-flag-title="TF-IDF算法">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读次数 </span>
<span class="leancloud-visitors-count"></span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="TF-IDF算法"><a href="#TF-IDF算法" class="headerlink" title="TF-IDF算法"></a>TF-IDF算法</h1><p>TF-IDF（词频-逆文档频率）算法是一种统计方法，用以评估一字词对于一个文件集或一个语料库中的其中一份文件的重要程度。<font color="ff0000">字词的重要性随着它在文件中出现的次数成正比增加，但同时会随着它在语料库中出现的频率成反比下降</font>。该算法在数据挖掘、文本处理和信息检索等领域得到了广泛的应用，如从一篇文章中找到它的关键词。</p>
<p>TFIDF的主要思想是：如果某个词或短语在一篇文章中出现的频率TF高，并且在其他文章中很少出现，则认为此词或者短语具有很好的类别区分能力，适合用来分类。TF-IDF实际上就是 TF*IDF，其中 TF（Term Frequency），表示词条在文章Document 中出现的频率；IDF（Inverse Document Frequency），其主要思想就是，如果包含某个词 Word的文档越少，则这个词的区分度就越大，也就是 IDF 越大。对于如何获取一篇文章的关键词，我们可以计算这边文章出现的所有名词的 TF-IDF，TF-IDF越大，则说明这个名词对这篇文章的区分度就越高，取 TF-IDF 值较大的几个词，就可以当做这篇文章的关键词。</p>
<h2 id="计算步骤"><a href="#计算步骤" class="headerlink" title="计算步骤"></a>计算步骤</h2><ol>
<li><p>计算词频（TF）</p>
<font size="6" face="微软雅黑">$词频=\frac{某个词在文章中的出现次数}{文章总次数}$</font>
</li>
<li><p>计算逆文档频率（IDF）</p>
<font size="6" face="微软雅黑">$逆文档频率=\log \frac{语料库的文档总数}{包含该词的文档数+1}$</font>
</li>
<li><p>计算词频-逆文档频率（TF-IDF）</p>
<font size="6" face="微软雅黑">$词频-逆文档频率={词频}\ast{逆文档频率}$</font>
</li>
</ol>
<h2 id="代码示例"><a href="#代码示例" class="headerlink" title="代码示例"></a>代码示例</h2><p>为了建立自己的语料库，我从网上爬了100个 url 存放在本地的 txt 文件中，<br>存放格式如下：<br><img src="http://img.blog.csdn.net/20161216234101427?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvbGlvbmVsX2Zlbmdq/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" alt="这里写图片描述"><br>这就相当于存放了100篇文章，即语料库的的文档总数是100。</p>
<ul>
<li>TF-IDF 算法代码</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br></pre></td><td class="code"><pre><span class="line">package com.myapp.ml.nlp;</span><br><span class="line"></span><br><span class="line">import org.ansj.domain.Term;</span><br><span class="line">import org.ansj.splitWord.analysis.ToAnalysis;</span><br><span class="line">import org.ansj.util.FilterModifWord;</span><br><span class="line">import org.apache.commons.lang3.StringUtils;</span><br><span class="line">import org.jsoup.Jsoup;</span><br><span class="line">import org.jsoup.nodes.Document;</span><br><span class="line"></span><br><span class="line">import java.io.BufferedReader;</span><br><span class="line">import java.io.FileReader;</span><br><span class="line">import java.io.IOException;</span><br><span class="line">import java.util.ArrayList;</span><br><span class="line">import java.util.List;</span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> * Created by lionel on 16/12/15.</span><br><span class="line"> */</span><br><span class="line">public class TFIDFAlgorithm {</span><br><span class="line"> /**</span><br><span class="line"> * 根据文件路径，文件中存放的100个网址的 url，获取 url 路径列表</span><br><span class="line"> *</span><br><span class="line"> * @param path 本地文件路径</span><br><span class="line"> * @return 路径列表</span><br><span class="line"> */</span><br><span class="line"> public List<String> readUrlFromText(String path) {</span><br><span class="line"> if (StringUtils.isBlank(path)) {</span><br><span class="line"> return null;</span><br><span class="line"> }</span><br><span class="line"> List<String> urls = new ArrayList<String>();</span><br><span class="line"> try {</span><br><span class="line"> BufferedReader reader = new BufferedReader(new FileReader(path));</span><br><span class="line"> String line;</span><br><span class="line"> while ((line = reader.readLine()) != null) {</span><br><span class="line"> urls.add(line.trim());</span><br><span class="line"> }</span><br><span class="line"> } catch (IOException e) {</span><br><span class="line"> e.printStackTrace();</span><br><span class="line"> }</span><br><span class="line"> return urls;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 利用 Jsoup 工具，根据网址获取网页文本</span><br><span class="line"> *</span><br><span class="line"> * @param url 网址</span><br><span class="line"> * @return 网页文本</span><br><span class="line"> */</span><br><span class="line"> public String getTextFromUrl(String url) {</span><br><span class="line"> if (StringUtils.isBlank(url)) {</span><br><span class="line"> return null;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> String text = "";</span><br><span class="line"> try {</span><br><span class="line"> Document document = Jsoup.connect(url).get();</span><br><span class="line"> text = document.text();</span><br><span class="line"> } catch (IOException e) {</span><br><span class="line"> e.printStackTrace();</span><br><span class="line"> }</span><br><span class="line"> return text.replace(" ", "");</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 运用 ansj 给文本分词</span><br><span class="line"> *</span><br><span class="line"> * @param text 文本内容</span><br><span class="line"> * @return 分词结果</span><br><span class="line"> */</span><br><span class="line"> public List<Term> parse(String text) {</span><br><span class="line"> if (StringUtils.isBlank(text)) {</span><br><span class="line"> return null;</span><br><span class="line"> }</span><br><span class="line"> List<Term> terms = FilterModifWord.modifResult(ToAnalysis.parse(text));</span><br><span class="line"> if (terms == null || terms.size() == 0) {</span><br><span class="line"> return null;</span><br><span class="line"> }</span><br><span class="line"> return terms;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 计算一篇文章分词后除去标点符号后词的总数</span><br><span class="line"> *</span><br><span class="line"> * @param terms 分词后的集合</span><br><span class="line"> * @return 一篇文章分词后除去标点符号后词的总数</span><br><span class="line"> */</span><br><span class="line"> private Integer countWord(List<Term> terms) {</span><br><span class="line"> if (terms == null || terms.size() == 0) {</span><br><span class="line"> return null;</span><br><span class="line"> }</span><br><span class="line"> for (int i = 0; i < terms.size(); i++) {</span><br><span class="line"> if ("null".equals(terms.get(i).getNatureStr()) || terms.get(i).getNatureStr().startsWith("w")) {</span><br><span class="line"> terms.remove(i);</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> return terms.size();</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 计算词频 IF</span><br><span class="line"> *</span><br><span class="line"> * @param word 词</span><br><span class="line"> * @param terms 分词结果集合</span><br><span class="line"> * @return IF</span><br><span class="line"> */</span><br><span class="line"> public double computeTF(String word, List<Term> terms) {</span><br><span class="line"> if (StringUtils.isBlank(word)) {</span><br><span class="line"> return 0.0;</span><br><span class="line"> }</span><br><span class="line"> int count = 0;</span><br><span class="line"> for (Term term : terms) {</span><br><span class="line"> if (term.getName().equals(word)) {</span><br><span class="line"> count += 1;</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> return (double) count / countWord(terms);</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 统计词语的逆文档频率 IDF</span><br><span class="line"> *</span><br><span class="line"> * @param path 存放 url 的文件路径</span><br><span class="line"> * @param word IDF</span><br><span class="line"> */</span><br><span class="line"> public double computeIDF(String path, String word) {</span><br><span class="line"> if (StringUtils.isBlank(path) || StringUtils.isBlank(word)) {</span><br><span class="line"> return 0.0;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> List<String> urls = readUrlFromText(path);</span><br><span class="line"> int count = 1;</span><br><span class="line"> for (String url : urls) {</span><br><span class="line"> String text = getTextFromUrl(url);</span><br><span class="line"> if (text.contains(word)) {</span><br><span class="line"> count += 1;</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> return Math.log10((double) urls.size() / count);</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 计算词频-逆文档频率 TF—IDF</span><br><span class="line"> *</span><br><span class="line"> * @param filePath 存放url的文件路径</span><br><span class="line"> * @param terms 分词结果集合</span><br><span class="line"> * @param word 词</span><br><span class="line"> * @return TF—IDF</span><br><span class="line"> */</span><br><span class="line"></span><br><span class="line"> public Double computeTFIDF(String filePath, List<Term> terms, String word) {</span><br><span class="line"> return computeTF(word, terms) * computeIDF(filePath, word);</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<ul>
<li>测试代码</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">package com.myapp.ml.nlp;</span><br><span class="line"></span><br><span class="line">import org.ansj.domain.Term;</span><br><span class="line">import org.junit.Test;</span><br><span class="line"></span><br><span class="line">import java.util.List;</span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> *测试词语“语言”的 TF-IDF</span><br><span class="line"> * Created by lionel on 16/12/15.</span><br><span class="line"> */</span><br><span class="line">public class TFIDFAlgorithmTest {</span><br><span class="line"> @Test</span><br><span class="line"> public void test() {</span><br><span class="line"> TFIDFAlgorithm tfidfAlgorithm = new TFIDFAlgorithm();</span><br><span class="line"> String filePath = "/Users/lionel/PycharmProjects/python-app/com/pythonapp/spider/output.txt";</span><br><span class="line"> String url = "http://baike.baidu.com/item/Java/85979";</span><br><span class="line"> String word = "语言";</span><br><span class="line"> List<Term> terms = tfidfAlgorithm.parse(tfidfAlgorithm.getTextFromUrl(url));</span><br><span class="line"> System.out.println("[【" + word + "】词频 ] " + tfidfAlgorithm.computeTF(word, terms));</span><br><span class="line"> System.out.println("[【" + word + "】逆文档频率 ] " + tfidfAlgorithm.computeIDF(filePath, word));</span><br><span class="line"> System.out.println("[【" + word + "】词频-逆文档频率 ] "+tfidfAlgorithm.computeTFIDF(filePath,terms,word));</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<ul>
<li>测试结果<br><img src="http://img.blog.csdn.net/20161217001245667?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvbGlvbmVsX2Zlbmdq/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" alt="这里写图片描述"></li>
</ul>
<p>本篇博客描述的是我对 TF-IDF 算法的理解，代码是对算法过程简单的实现，若有失偏颇，还请指出。</p>
</div>
<div>
</div>
<div>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
<link itemprop="mainEntityOfPage" href="http://JiangFeng.github.com/2018/01/16/java并发编程-volatile关键字/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Jiang Feng">
<meta itemprop="description" content="">
<meta itemprop="image" content="/images/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="江峰的技术博客">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2018/01/16/java并发编程-volatile关键字/" itemprop="url">
java并发编程-volatile关键字
</a>
</h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建于" itemprop="dateCreated datePublished" datetime="2018-01-16T17:45:52+08:00">
2018-01-16
</time>
</span>
<span class="post-category" >
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-folder-o"></i>
</span>
<span class="post-meta-item-text">分类于</span>
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/并发编程/" itemprop="url" rel="index">
<span itemprop="name">并发编程</span>
</a>
</span>
</span>
<span id="/2018/01/16/java并发编程-volatile关键字/" class="leancloud_visitors" data-flag-title="java并发编程-volatile关键字">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读次数 </span>
<span class="leancloud-visitors-count"></span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<ul>
<li><p>可见性</p>
<p> 使用volatile来修饰共享变量时，就保证了不同线程对这个变量操作的可见性，也就是说，一个线程如果修改了这个变量的值，会及时把改变后的值更新到主内存中，对其它线程来说是可见的。</p>
</li>
<li><p>有序性</p>
<p> 由于volatile关键字能禁止指令重排序，所以保证了“一定”的有序性。</p>
</li>
<li><p>原子性</p>
<p> 由上可知，volatile关键字可以保证可见性和“一定”的有序性，那么volatile关键字能保证对变量操作的原子性吗？首先来看下下面的例子。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line">import java.util.concurrent.locks.Lock;</span><br><span class="line">import java.util.concurrent.locks.ReentrantLock;</span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> * Created by lionel on 18/1/12.</span><br><span class="line"> */</span><br><span class="line">public class VolatileTest {</span><br><span class="line"> public volatile int inc = 0;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> Lock lock = new ReentrantLock();</span><br><span class="line"></span><br><span class="line"> public void increase() {</span><br><span class="line"> inc++;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> public static void main(String[] args) {</span><br><span class="line"> final VolatileTest volatileTest = new VolatileTest();</span><br><span class="line"> for (int i = 0; i < 10; i++) {</span><br><span class="line"> new Thread() {</span><br><span class="line"> public void run() {</span><br><span class="line"> for (int j = 0; j < 1000; j++) {</span><br><span class="line"> volatileTest.increase();</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> }.start();</span><br><span class="line"> }</span><br><span class="line"> try {</span><br><span class="line"> Thread.sleep(1000);</span><br><span class="line"> } catch (InterruptedException e) {</span><br><span class="line"> e.printStackTrace();</span><br><span class="line"> }</span><br><span class="line"> System.out.println(volatileTest.inc);</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p> 这个代码执行的结果是多少了?10000（可能有人认为由于 volatile 保证了变量的可见性，每次线程执行完自加操作后别的线程都能拿到，所以10个线程分别进行了1000词操作，最终的结果是10000）？事实上是每次执行的结果都不大一样，且每次结果都小于10000。为什么会这样<br> 了？这是因为volatile关键字无法保证对变量操作的原子性造成的。由于自增操作是“复合操作”（读取变量原始值、加1操作、写入工作内存）。比方说某个时刻变量inc的值为10，线程1对变量进行自增操作，线程1先读取了变量inc的原始值，然后线程1被阻塞了；然后线程2对变量进行自增操作，线程2也去读取变量inc的原始值，由于线程1只是对变量inc进行读取操作，并没有对变量进行修改操作，所以此时主内存的值并没有改变，依然是10。而此时，线程2发现inc的值时10，然后进行加1操作，并把11写入工作内存，最后写入主存。然后线程1接着进行加1操作，由于已经读取了inc的值，注意此时在线程1的工作内存中inc的值仍然为10，所以线程1对inc进行加1操作后inc的值为11，然后将11写入工作内存，最后写入主存。那么两个线程分别进行了一次自增操作后，inc只增加了1。<br> <br><br> 怎样可以保证代码的原子性了，使用synchronized或者Lock。<br> 代码如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br></pre></td><td class="code"><pre><span class="line">import java.util.concurrent.locks.Lock;</span><br><span class="line">import java.util.concurrent.locks.ReentrantLock;</span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> * Created by lionel on 18/1/12.</span><br><span class="line"> */</span><br><span class="line">public class VolatileTest {</span><br><span class="line"> public volatile int inc = 0;</span><br><span class="line"></span><br><span class="line"> Lock lock = new ReentrantLock();</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * volatile 无法保证对变量操作是原子性的</span><br><span class="line"> */</span><br><span class="line"> public void increase() {</span><br><span class="line"> inc++;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 方案一:synchronized</span><br><span class="line"> */</span><br><span class="line"> public synchronized void increase1() {</span><br><span class="line"> inc++;</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 方案二:ReentrantLock</span><br><span class="line"> */</span><br><span class="line"> public void increase2() {</span><br><span class="line"> lock.lock();</span><br><span class="line"> try {</span><br><span class="line"> inc++;</span><br><span class="line"> } finally {</span><br><span class="line"> lock.unlock();</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> public static void main(String[] args) {</span><br><span class="line"> final VolatileTest volatileTest = new VolatileTest();</span><br><span class="line"> for (int i = 0; i < 10; i++) {</span><br><span class="line"> new Thread() {</span><br><span class="line"> public void run() {</span><br><span class="line"> for (int j = 0; j < 1000; j++) {</span><br><span class="line">// volatileTest.increase();</span><br><span class="line"> volatileTest.increase1();//使用 synchronized</span><br><span class="line">// volatileTest.increase2();//使用 Lock</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> }.start();</span><br><span class="line"> }</span><br><span class="line"> try {</span><br><span class="line"> Thread.sleep(1000);</span><br><span class="line"> } catch (InterruptedException e) {</span><br><span class="line"> e.printStackTrace();</span><br><span class="line"> }</span><br><span class="line"> System.out.println(volatileTest.inc);</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
</li>
</ul>
</div>
<div>
</div>
<div>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
<link itemprop="mainEntityOfPage" href="http://JiangFeng.github.com/2018/01/16/java内存模型/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Jiang Feng">
<meta itemprop="description" content="">
<meta itemprop="image" content="/images/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="江峰的技术博客">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2018/01/16/java内存模型/" itemprop="url">
java内存模型
</a>
</h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建于" itemprop="dateCreated datePublished" datetime="2018-01-16T17:45:38+08:00">
2018-01-16
</time>
</span>
<span id="/2018/01/16/java内存模型/" class="leancloud_visitors" data-flag-title="java内存模型">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读次数 </span>
<span class="leancloud-visitors-count"></span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p><img src="http://onm4pqoqp.bkt.clouddn.com/java%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B.png" alt="java内存模型"></p>
<p> Java线程之间的通信采用的是通过共享内存模型-java 内存模型（简称JMM）,<font color="red">JMM决定一个线程对共享变量的写入何时对另一个线程可见</font>。从抽象的角度来看，JMM定义了线程和主内存之间的抽象关系：<font color="red">线程之间的共享变量存储在主内存（main memory）中，每个线程都有一个私有的本地内存（local memory），本地内存中存储了该线程以读/写共享变量的副本</font>。</p>
<p> 从上图可以看出，线程A和线程B进行通信要经历以下两个步骤：<br>（1）线程A把本地内存A中更新过的共享变量刷新到主内存中去。（2）线程B到主内存中去读取线程A之前已更新过的共享变量。<br><img src="http://onm4pqoqp.bkt.clouddn.com/java%E5%86%85%E5%AD%98%E6%A8%A1%E5%9E%8B%E7%A4%BA%E4%BE%8B.png" alt=""></p>
<p> 如上图所示，本地内存A和B有主内存中共享变量x的副本。设初始时x都为0，线程A把更新后的值x=1临时存放在本地内存中。当线程 A 和线程 B 需要通信时，线程 A 首先会把自己本地内存中修改后的值刷新到主内存中，此时主内存中的 x 也为1了。然后，线程 B 会到主内存中去读取线程 A 更新后的值，此时线程 B 的本地内存也就变为1了。</p>
</div>
<div>
</div>
<div>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
<link itemprop="mainEntityOfPage" href="http://JiangFeng.github.com/2018/01/16/RNN实现手写数字识别/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Jiang Feng">
<meta itemprop="description" content="">
<meta itemprop="image" content="/images/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="江峰的技术博客">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2018/01/16/RNN实现手写数字识别/" itemprop="url">
RNN实现手写数字识别
</a>
</h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建于" itemprop="dateCreated datePublished" datetime="2018-01-16T17:44:25+08:00">
2018-01-16
</time>
</span>
<span class="post-category" >
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-folder-o"></i>
</span>
<span class="post-meta-item-text">分类于</span>
<span itemprop="about" itemscope itemtype="http://schema.org/Thing">
<a href="/categories/深度学习/" itemprop="url" rel="index">
<span itemprop="name">深度学习</span>
</a>
</span>
</span>
<span id="/2018/01/16/RNN实现手写数字识别/" class="leancloud_visitors" data-flag-title="RNN实现手写数字识别">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读次数 </span>
<span class="leancloud-visitors-count"></span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">