-
Notifications
You must be signed in to change notification settings - Fork 1
/
index.html
192 lines (192 loc) · 69.9 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
<!DOCTYPE html><html><head><meta name="generator" content="Hexo 3.8.0"><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"><meta name="description" content><meta name="keywords" content><meta name="author" content="warmqing"><meta name="copyright" content="warmqing"><meta name="google-site-verification" content="r6fMmphB-u3s7p5myuSngUw4xAgHUz0nII5QYBUBG6c"><meta name="baidu-site-verification" content="5cSbHnUUCL"><meta name="sogou_site_verification" content="MF20RUhTSF"><title>Would you like some Oolong | Oolong Box</title><link rel="shortcut icon" href="/melody-favicon.ico"><link rel="stylesheet" href="/css/index.css?version=1.6.1"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/font-awesome@latest/css/font-awesome.min.css?version=1.6.1"><link rel="dns-prefetch" href="https://cdn.staticfile.org"><link rel="dns-prefetch" href="https://cdn.bootcss.com"><link rel="dns-prefetch" href="https://creativecommons.org"><link rel="dns-prefetch" href="https://cdn.jsdelivr.net"><link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/instantsearch.min.css"><script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/instantsearch.min.js" defer></script><link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/gitalk/dist/gitalk.min.css"><script src="https://cdn.jsdelivr.net/npm/gitalk@latest/dist/gitalk.min.js"></script><script src="https://cdn.jsdelivr.net/npm/[email protected]/js/md5.min.js"></script><script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script><script>(adsbygoogle = window.adsbygoogle || []).push({
google_ad_client: 'ca-pub-4146880878125243',
enable_page_level_ads: 'true'
});
</script><link rel="dns-prefetch" href="https://hm.baidu.com"><script>var _hmt = _hmt || [];
(function() {
var hm = document.createElement("script");
hm.src = "https://hm.baidu.com/hm.js?a30a198fc71d7b0aa216b4f7f31698a2";
var s = document.getElementsByTagName("script")[0];
s.parentNode.insertBefore(hm, s);
})();</script><script>var GLOBAL_CONFIG = {
root: '/',
algolia: {"appId":"V8UUR9G3TF","apiKey":"5a765edf66f2abc7c0a86abdac87b2de","indexName":"blog_index","hits":{"per_page":10},"languages":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}.","hits_stats":"${hits} results found in ${time} ms"}},
localSearch: undefined,
copy: {
success: 'Copy successfully',
error: 'Copy error',
noSupport: 'The browser does not support'
}
} </script></head><body><i class="fa fa-arrow-right" id="toggle-sidebar" aria-hidden="true"></i><div id="sidebar"><div class="author-info"><div class="author-info__avatar text-center"><img src="/img/avatar.png"></div><div class="author-info__name text-center">warmqing</div><div class="author-info__description text-center"></div><hr><div class="author-info-articles"><a class="author-info-articles__archives article-meta" href="/archives"><span class="pull-left">Articles</span><span class="pull-right">15</span></a><a class="author-info-articles__tags article-meta" href="/tags"><span class="pull-left">Tags</span><span class="pull-right">8</span></a><a class="author-info-articles__categories article-meta" href="/categories"><span class="pull-left">Categories</span><span class="pull-right">3</span></a></div></div></div><nav id="nav" style="background-image: url(https://i.loli.net/2019/04/04/5ca58c8caa281.jpg)"><div id="page-header"><span class="pull-left"> <a id="site-name" href="/">Oolong Box</a></span><i class="fa fa-bars toggle-menu pull-right" aria-hidden="true"></i><span class="pull-right menus"><a class="site-page social-icon search"><i class="fa fa-search"></i><span> Search</span></a><a class="site-page" href="/">Home</a><a class="site-page" href="/archives">Archives</a><a class="site-page" href="/tags">Tags</a><a class="site-page" href="/categories">Categories</a></span></div><div id="site-info"><div id="site-title">Oolong Box</div><div id="site-sub-title">Would you like some Oolong</div><div id="site-social-icons"><a class="social-icon" href="https://github.com/warmqing"><i class="fa-github fa"></i></a><a class="social-icon" href="https://warmqing.github.io/atom.xml"><i class="fa-rss fa"></i></a><a class="social-icon search"><i class="fa fa-search"></i></a></div></div></nav><div id="content-outer"><div class="layout" id="content-inner"><div class="recent-post-item article-container"><a class="article-title" href="/box/9b322e8a/">Scala的实际应用</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-07-06</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/scala/">scala</a></span><div class="content"><h1 id="Scala的实际应用"><a href="#Scala的实际应用" class="headerlink" title="Scala的实际应用"></a>Scala的实际应用</h1><h2 id="订单实体类"><a href="#订单实体类" class="headerlink" title="订单实体类"></a>订单实体类</h2><ul>
<li><p>私有成员变量，getter/setter方法</p>
</li>
<li><p>在Scala中，运算符即是方法，运算符有优先级之分</p>
</li>
<li><p>使用字符串插值器进行Json转换</p>
</li>
</ul>
<figure class="highlight scala"><table><tr><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">OrderVo</span> <span class="keyword">extends</span> <span class="title">Serializable</span> </span>{</span><br><span class="line"> <span class="keyword">private</span> <span class="keyword">var</span> _orderId: <span class="type">Long</span> = <span class="number">0</span></span><br><span class="line"> <span class="keyword">private</span> <span class="keyword">var</span> _userId: <span class="type">Long</span> = <span class="number">0</span></span><br><span class="line"> <span class="keyword">private</span> <span class="keyword">var</span> _orderTime: <span class="type">String</span> = <span class="literal">null</span></span><br><span class="line"> <span class="keyword">private</span> <span class="keyword">var</span> _gmv: <span class="type">BigDecimal</span> = <span class="number">0.0</span></span><br><span class="line"> <span class="keyword">private</span> <span class="keyword">var</span> _validNum: <span class="type">Long</span> = <span class="number">0</span></span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">orderId</span> </span>= _orderId</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">orderId_=</span></span>(v: <span class="type">Long</span>) = {</span><br><span class="line"> _orderId = v</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">userId</span> </span>= _userId</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">userId_=</span></span>(v: <span class="type">Long</span>) = {</span><br><span class="line"> _userId = v</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">orderTime</span> </span>= _orderTime</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">orderTime_=</span></span>(v: <span class="type">String</span>) = {</span><br><span class="line"> _orderTime = v</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">gmv</span> </span>= _gmv</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">gmv_=</span></span>(v: <span class="type">BigDecimal</span>) = {</span><br><span class="line"> _gmv = v</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">validNum</span> </span>= _validNum</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">validNum_=</span></span>(v: <span class="type">Long</span>) = {</span><br><span class="line"> _validNum = v</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 使用操作符+作为方法名</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * @param order</span></span><br><span class="line"><span class="comment"> * @return</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">+</span></span>(order: <span class="type">OrderVo</span>): <span class="type">OrderVo</span> = {</span><br><span class="line"> <span class="keyword">if</span> (order != <span class="literal">null</span>) {</span><br><span class="line"> gmv = gmv + order.gmv</span><br><span class="line"> validNum = validNum + order.validNum</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">this</span></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">*</span></span>(order: <span class="type">OrderVo</span>): <span class="type">OrderVo</span> = {</span><br><span class="line"> <span class="keyword">if</span> (order != <span class="literal">null</span>) {</span><br><span class="line"> gmv = gmv * order.gmv</span><br><span class="line"> validNum = validNum * order.validNum</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">this</span></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="keyword">override</span> <span class="function"><span class="keyword">def</span> <span class="title">toString</span></span>: <span class="type">String</span> = {</span><br><span class="line"> <span class="string">s""</span><span class="string">""</span>userI<span class="string">d":<span class="subst">$userId</span>,"</span>orderI<span class="string">d":<span class="subst">$orderId</span>,"</span>orderT<span class="string">ime":"</span>$orderT<span class="string">ime","</span>validN<span class="string">um":<span class="subst">$validNum</span>,"</span><span class="string">gmv":<span class="subst">$gmv</span>"</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">object</span> <span class="title">OrderVo</span> </span>{</span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">fromString</span></span>(in: <span class="type">String</span>): <span class="type">OrderVo</span> ={</span><br><span class="line"> <span class="keyword">val</span> json = <span class="type">JSON</span>.parseObject(<span class="string">s"{<span class="subst">$in</span>}"</span>)</span><br><span class="line"> <span class="keyword">if</span> (json == <span class="literal">null</span> || json.isEmpty) {</span><br><span class="line"> <span class="literal">null</span></span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="keyword">val</span> o = <span class="keyword">new</span> <span class="type">OrderVo</span></span><br><span class="line"> o.userId = json.getLong(<span class="string">"userId"</span>)</span><br><span class="line"> o.orderId = json.getLong(<span class="string">"orderId"</span>)</span><br><span class="line"> o.orderTime = json.getString(<span class="string">"orderTime"</span>)</span><br><span class="line"> o.validNum = json.getLong(<span class="string">"validNum"</span>)</span><br><span class="line"> o.gmv = json.getBigDecimal(<span class="string">"gmv"</span>)</span><br><span class="line"> o</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div><a class="more" href="/box/9b322e8a/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/7f120df8/">Spark Streaming实时汇总订单量、GMV</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-06-18</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/scala/">scala</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/spark/">spark</a></span><div class="content"><h1 id="Spark-Streaming实时汇总订单量、GMV"><a href="#Spark-Streaming实时汇总订单量、GMV" class="headerlink" title="Spark Streaming实时汇总订单量、GMV"></a>Spark Streaming实时汇总订单量、GMV</h1><h2 id="整体思路"><a href="#整体思路" class="headerlink" title="整体思路"></a>整体思路</h2><p><img src="https://i.loli.net/2020/06/17/ZSAkputjf83xGMW.png" alt></p>
<h2 id="订单变化基差"><a href="#订单变化基差" class="headerlink" title="订单变化基差"></a>订单变化基差</h2><p><strong>基差</strong>是某一特定商品于当前时刻的价格与上一时刻价格之差。计算方法是使用当前价格减去历史价格。若当前价格低于历史价格，基差为负值；当前价格高于历史价格，基差为正值。</p>
<p><strong>订单变化</strong>，此处仅列举一些简单的场景，当订单由多件sku（商品）组成，发生单件退换货时，会导致订单金额的扣除，当订单下单后取消，或者完成后被风控判定无效，会导致订单状态由“有效”变为“无效”。</p>
<p>使用基差的方式，将订单变化的过程转发为Kafka中的消息体，如下图中的订单变化，分别会发送Message1、Message2、Message3三条消息</p>
<p><img src="https://i.loli.net/2020/06/18/ipXb18LIlP3AWEf.png" alt></p></div><a class="more" href="/box/7f120df8/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/df0ed28e/">JVM——内存分配与回收策略</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-06-17</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/JVM/">JVM</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/Java/">Java</a></span><div class="content"><h1 id="JVM——内存分配与回收策略"><a href="#JVM——内存分配与回收策略" class="headerlink" title="JVM——内存分配与回收策略"></a>JVM——内存分配与回收策略</h1><h2 id="内存分配策略"><a href="#内存分配策略" class="headerlink" title="内存分配策略"></a>内存分配策略</h2><h2 id="●-对象优先在-Eden-分配"><a href="#●-对象优先在-Eden-分配" class="headerlink" title="● 对象优先在 Eden 分配"></a>● 对象优先在 Eden 分配</h2><p>大多数情况下，对象在新生代 Eden 上分配，当 Eden 空间不够时，发起 Minor GC。</p>
<h3 id="●-大对象直接进入老年代"><a href="#●-大对象直接进入老年代" class="headerlink" title="● 大对象直接进入老年代"></a>● 大对象直接进入老年代</h3><p>大对象是指需要连续内存空间的对象，最典型的大对象是那种很长的字符串以及数组。</p>
<p>经常出现大对象会提前触发垃圾收集以获取足够的连续空间分配给大对象。</p>
<p>-XX:PretenureSizeThreshold，大于此值的对象直接在老年代分配，避免在 Eden 和 Survivor 之间的大量内存复制。</p>
<h3 id="●-长期存活的对象进入老年代"><a href="#●-长期存活的对象进入老年代" class="headerlink" title="● 长期存活的对象进入老年代"></a>● 长期存活的对象进入老年代</h3><p>为对象定义年龄计数器，对象在 Eden 出生并经过 Minor GC 依然存活，将移动到 Survivor 中，年龄就增加 1 岁，增加到一定年龄则移动到老年代中。</p>
<p>-XX:MaxTenuringThreshold 用来定义年龄的阈值。</p>
<h3 id="●-动态对象年龄判定"><a href="#●-动态对象年龄判定" class="headerlink" title="● 动态对象年龄判定"></a>● 动态对象年龄判定</h3><p>虚拟机并不是永远要求对象的年龄必须达到 MaxTenuringThreshold 才能晋升老年代，如果在 Survivor 中相同年龄所有对象大小的总和大于 Survivor 空间的一半，则年龄大于或等于该年龄的对象可以直接进入老年代，无需等到 MaxTenuringThreshold 中要求的年龄。</p>
<h3 id="●-空间分配担保"><a href="#●-空间分配担保" class="headerlink" title="● 空间分配担保"></a>● 空间分配担保</h3><p>在发生 Minor GC 之前，虚拟机先检查老年代最大可用的连续空间是否大于新生代所有对象总空间，如果条件成立的话，那么 Minor GC 可以确认是安全的。</p>
<p>如果不成立的话虚拟机会查看 HandlePromotionFailure 的值是否允许担保失败，如果允许那么就会继续检查老年代最大可用的连续空间是否大于历次晋升到老年代对象的平均大小，如果大于，将尝试着进行一次 Minor GC；如果小于，或者 HandlePromotionFailure 的值不允许冒险，那么就要进行一次 Full GC。</p>
<h2 id="Minor-GC-Young-GC、Major-GC-Full-FC"><a href="#Minor-GC-Young-GC、Major-GC-Full-FC" class="headerlink" title="Minor GC/Young GC、Major GC/Full FC"></a>Minor GC/Young GC、Major GC/Full FC</h2><h3 id="●-新生代-GC（Minor-GC-Young-GC）"><a href="#●-新生代-GC（Minor-GC-Young-GC）" class="headerlink" title="● 新生代 GC（Minor GC/Young GC）"></a>● 新生代 GC（Minor GC/Young GC）</h3><p>从年轻代空间（包括 Eden 和 Survivor 区域）回收内存被称为 Minor GC,因为 Java 对象大多都具备朝生夕灭的特性，所以 Minor GC 非常频繁，一般回收速度也比较快。</p>
<ol>
<li>当 JVM 无法为一个新的对象分配空间时会触发 Minor GC，比如当 Eden 区满了。所以分配率越高，越频繁执行 Minor GC。</li>
<li>内存池被填满的时候，其中的内容全部会被复制，指针会从0开始跟踪空闲内存。Eden 和 Survivor 区进行了标记和复制操作，取代了经典的标记、扫描、压缩、清理操作。所以 Eden 和 Survivor 区不存在内存碎片。写指针总是停留在所使用内存池的顶部。</li>
<li>执行 Minor GC 操作时，不会影响到永久代。从永久代到年轻代的引用被当成 GC roots，从年轻代到永久代的引用在标记阶段被直接忽略掉。</li>
<li>质疑常规的认知，所有的 Minor GC 都会触发“全世界的暂停（stop-the-world）”，停止应用程序的线程。对于大部分应用程序，停顿导致的延迟都是可以忽略不计的。其中的真相就 是，大部分 Eden 区中的对象都能被认为是垃圾，永远也不会被复制到 Survivor 区或者老年代空间。如果正好相反，Eden 区大部分新生对象不符合 GC 条件，Minor GC 执行时暂停的时间将会长很多。</li>
</ol>
<h3 id="●-Minor-GC触发机制："><a href="#●-Minor-GC触发机制：" class="headerlink" title="● Minor GC触发机制："></a>● Minor GC触发机制：</h3><p>当年轻代满时就会触发Minor GC，这里的年轻代满指的是Eden代满，Survivor满不会引发GC。</p>
<p>新生代分为一个 Eden区和两个Survivor区的垃圾收集叫做 Minor GC。 清除 Eden 和 from ，转到to中。接下来from与to转换。继续清除 Eden和新from，转到to。清除一次后存活超过年龄的 ，转到老年代。to到了阈值后，部分对象转到老年代。</p>
<h3 id="●-老年代-GC（Major-GC-Full-GC）"><a href="#●-老年代-GC（Major-GC-Full-GC）" class="headerlink" title="● 老年代 GC（Major GC / Full GC）"></a>● 老年代 GC（Major GC / Full GC）</h3><p>指发生在老年代的 GC，出现了 Major GC，经常会伴随至少一次的 Minor GC（但非绝对的，ParallelScavenge 收集器的收集策略里就有直接进行 Major GC 的策略选择过程） 。MajorGC 的速度一般会比 Minor GC 慢 10倍以上。</p></div><a class="more" href="/box/df0ed28e/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/a25844fe/">JVM——类加载机制</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-06-17</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/JVM/">JVM</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/Java/">Java</a></span><div class="content"><h1 id="JVM——类加载机制"><a href="#JVM——类加载机制" class="headerlink" title="JVM——类加载机制"></a>JVM——类加载机制</h1><h2 id="类的生命周期"><a href="#类的生命周期" class="headerlink" title="类的生命周期"></a>类的生命周期</h2><p><img src="https://i.loli.net/2020/06/16/kC9ieRA86UbpG7B.png" alt></p>
<p>包括以下 7 个阶段：</p>
<ul>
<li><strong>加载（Loading）</strong></li>
<li><strong>验证（Verification）</strong></li>
<li><strong>准备（Preparation）</strong></li>
<li><strong>解析（Resolution）</strong></li>
<li><strong>初始化（Initialization）</strong></li>
<li>使用（Using）</li>
<li>卸载（Unloading）</li>
</ul>
<h2 id="类初始化时机"><a href="#类初始化时机" class="headerlink" title="类初始化时机"></a>类初始化时机</h2><h3 id="1-主动引用"><a href="#1-主动引用" class="headerlink" title="1. 主动引用"></a>1. 主动引用</h3><p><strong>有且仅有 5 种情况必须立即对类进行“初始化”</strong>：（加载、验证、准备都会随之发生）：</p>
<ul>
<li>遇到 new、getstatic、putstatic、invokestatic 这四条字节码指令时，如果类没有进行过初始化，则必须先触发其初始化。最常见的生成这 4 条指令的场景是：使用 new 关键字实例化对象的时候；读取或设置一个类的静态字段（被 final 修饰、已在编译期把结果放入常量池的静态字段除外）的时候；以及调用一个类的静态方法的时候。</li>
<li>使用 java.lang.reflect 包的方法对类进行反射调用的时候，如果类没有进行初始化，则需要先触发其初始化。</li>
<li>当初始化一个类的时候，如果发现其父类还没有进行过初始化，则需要先触发其父类的初始化。</li>
<li>当虚拟机启动时，用户需要指定一个要执行的主类（包含 main() 方法的那个类），虚拟机会先初始化这个主类；</li>
<li>当使用 JDK 1.7 的动态语言支持时，如果一个 java.lang.invoke.MethodHandle 实例最后的解析结果为 REF_getStatic, REF_putStatic, REF_invokeStatic 的方法句柄，并且这个方法句柄所对应的类没有进行过初始化，则需要先触发其初始化；</li>
</ul>
<p>这 5 种场景中的行为称为对一个类进行<strong>主动引用</strong>，除此之外，所有引用类的方式都不会触发初始化，称为<strong>被动引用</strong>。</p>
<h3 id="2-被动引用"><a href="#2-被动引用" class="headerlink" title="2. 被动引用"></a>2. 被动引用</h3><ul>
<li>通过子类引用父类的静态字段，不会导致子类初始化。对于静态字段，只有直接定义这个字段的类才会被初始化。</li>
</ul>
<figure class="highlight plain"><table><tr><td class="code"><pre><span class="line">System.out.println(SubClass.value); // value 字段在 SuperClass 中定义</span><br></pre></td></tr></table></figure>
<ul>
<li>通过数组定义来引用类，不会触发此类的初始化。该过程会对数组类进行初始化，数组类是一个由虚拟机自动生成的、直接继承自 Object 的子类，其中包含了数组的属性和方法。</li>
</ul>
<figure class="highlight plain"><table><tr><td class="code"><pre><span class="line">SuperClass[] sca = new SuperClass[10];</span><br></pre></td></tr></table></figure>
<ul>
<li>常量在编译阶段会存入调用类的常量池中，本质上并没有直接引用到定义常量的类，因此不会触发定义常量的类的初始化。</li>
</ul>
<figure class="highlight plain"><table><tr><td class="code"><pre><span class="line">class ConstClass {</span><br><span class="line"> static {</span><br><span class="line"> System.out.println("ConstClass init!");</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> public static final String HELLO_WORLD = "Hello world";</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">System.out.println(ConstClass.HELLO_WORLD);</span><br></pre></td></tr></table></figure></div><a class="more" href="/box/a25844fe/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/a9fa83bb/">JVM——内存模型</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-06-17</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/JVM/">JVM</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/Java/">Java</a></span><div class="content"><h1 id="JVM——内存模型"><a href="#JVM——内存模型" class="headerlink" title="JVM——内存模型"></a>JVM——内存模型</h1><h2 id="JDK6"><a href="#JDK6" class="headerlink" title="JDK6"></a>JDK6</h2><p><img src="https://i.loli.net/2020/06/03/A9fGOk75RK6ezCE.png" alt></p>
<h2 id="JDK8"><a href="#JDK8" class="headerlink" title="JDK8"></a>JDK8</h2><p><img src="https://i.loli.net/2020/06/03/Xx1caoJfMNwdK59.png" alt></p>
<p> </p>
<h2 id="一、JVM内存模型分类"><a href="#一、JVM内存模型分类" class="headerlink" title="一、JVM内存模型分类"></a>一、JVM内存模型分类</h2><p>JVM内存模型从线程维度归类分为：<strong>线程私有内存</strong>、<strong>线程共享内存</strong>、以及不在堆内的<strong>直接内存</strong>。</p>
<h3 id="直接内存"><a href="#直接内存" class="headerlink" title="直接内存"></a>直接内存</h3><p><code>直接内存并不是JVM运行时数据区的一部分</code>,在JDK 1.4引入的NIO提供了基于Channel与Buffer的IO方式，它可以使用Native函数库直接分配堆外的内存，然后使用DirectByteBuffer对象作为这块内存的引用进行操作，避免了Java堆和Native堆来回拷贝数据，<code>因此在一些场景中可以显著提高性能。</code></p>
<p><strong>直接内存的分配不会受到Java堆大小的限制</strong>，（即不会遵守-Xms、-Xmx等配置）。但仍然受到本机总内存大小+寻址空间的限制，因此扩展时也会出现<code>OutOfMemoryError</code>异常。</p>
<h3 id="线程私有型内存"><a href="#线程私有型内存" class="headerlink" title="线程私有型内存"></a>线程私有型内存</h3><p><strong>程序寄存器（PC Register）、Java栈（Stack）、本地方法栈（Native Stack）</strong></p>
<h3 id="线程共享型内存"><a href="#线程共享型内存" class="headerlink" title="线程共享型内存"></a>线程共享型内存</h3><p><strong>Java堆(Heap)、本地方法区(Method Area)</strong></p>
<h2 id="二、程序计数器-Program-Counter-Register"><a href="#二、程序计数器-Program-Counter-Register" class="headerlink" title="二、程序计数器 - Program Counter Register"></a>二、程序计数器 - Program Counter Register</h2><p><strong>程序计数器（Program Counter Register）是一块较小的内存空间</strong>，<code>它的作用可以看做是当前线程所执行的字节码的行号指示器</code>。在虚拟机的概念模型里，字节码解释器工作时就是通过改变这个计数器的值来选取下一条需要执行的字节码指令，分支、循环、跳转、异常处理、线程恢复等基础功能都需要依赖这个计数器来完成。</p>
<p>由于Java虚拟机的多线程是通过线程轮流切换并分配处理器执行时间的方式来实现的，在任何一个确定的时刻，一个处理器（对于多核处理器来说是一个内核）只会执行一条线程中的指令。因此，<code>为了线程切换后能恢复到正确的执行位置，每条线程都需要有一个独立的程序计数器</code>，各条线程之间的计数器互不影响，独立存储，我们称这类内存区域为<code>“线程私有”</code>的内存。</p>
<p>如果线程正在执行的是一个Java方法，这个<code>计数器记录的是正在执行的虚拟机字节码指令的地址</code>；如果正在执行的是Natvie方法，这个计数器值则为空（Undefined）。</p>
<p>此内存区域是<code>唯一</code>一个在Java虚拟机规范中没有规定任何<code>OutOfMemoryError</code>情况的区域。</p>
<h2 id="三、Java栈-Java-Stack"><a href="#三、Java栈-Java-Stack" class="headerlink" title="三、Java栈 - Java Stack"></a>三、Java栈 - Java Stack</h2><p><strong>Java栈 - Java Stack</strong> Java虚拟机栈（Java Virtual Machine Stacks）也是<code>线程私有</code>的，它的<code>生命周期与线程相同</code>。虚拟机栈描述的是Java方法执行的内存模型：每个方法被执行的时候都会同时创建一个<code>栈帧（Stack Frame）</code>用于存储<code>局部变量表、操作栈、动态链接、方法出口</code>等信息。每一个方法被调用直至执行完成的过程，就对应着一个栈帧在虚拟机栈中从入栈到出栈的过程。</p>
<p>在Java虚拟机规范中，对这个区域规定了两种异常状况：</p>
<ul>
<li>如果线程请求的栈深度大于虚拟机所允许的深度，将抛出<code>StackOverflowError</code>异常；</li>
<li>如果虚拟机栈可以动态扩展（当前大部分的Java虚拟机都可动态扩展，只不过Java虚拟机规范中也允许固定长度的虚拟机栈），当扩展时无法申请到足够的内存时会抛出<code>OutOfMemoryError</code>异常。</li>
</ul></div><a class="more" href="/box/a9fa83bb/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/2aa0380c/">JVM——CMS、G1收集器</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2020-06-17</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/JVM/">JVM</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/Java/">Java</a></span><div class="content"><h1 id="JVM——CMS、G1收集器"><a href="#JVM——CMS、G1收集器" class="headerlink" title="JVM——CMS、G1收集器"></a>JVM——CMS、G1收集器</h1><h2 id="CMS回收垃圾的4个阶段"><a href="#CMS回收垃圾的4个阶段" class="headerlink" title="CMS回收垃圾的4个阶段"></a>CMS回收垃圾的4个阶段</h2><ol>
<li>初始标记</li>
<li>并发标记</li>
<li>重新标记</li>
<li>并发清理</li>
</ol>
<p><img src="https://i.loli.net/2020/05/25/RTXYaGhwOQIeyjt.jpg" alt></p>
<p>初始标记阶段：（CMS initial mark）标记GC Roots能直接关联的对象，时间短用户线程停顿，也就是 Stop the World 状态</p>
<p>并发标记阶段：（CMS concurrent mark）GC Roots直接关联对象向下遍历整个对象图，时间长不需要用户线程停顿，这个阶段最耗费时, 但这个阶段是和系统并发运行的，所以不会对系统运行造成影响</p>
<p>重新标记阶段：（CMS remark）由于第二阶段是并发执行的，一边标记垃圾对象，一边创建新对象，老对象会变成垃圾对象。 所以第三阶段也会进入 Stop the World 状态，并且重新标记，标记的是第二阶段中变动过的少数对象，所以运行速度很快</p>
<p>并发清理阶段：（CMS concurrent sweep） 清理删除掉标记阶段判断已经死亡的对象, 这个阶段也是会耗费很多时间，但由于是并发运行的，所以对系统不会造成很大的影响</p>
<h2 id="CMS优缺点"><a href="#CMS优缺点" class="headerlink" title="CMS优缺点"></a><strong>CMS优缺点</strong></h2><p>CMS采用 标记-清理 的算法，标记出垃圾对象，清除垃圾对象。算法是基于老年代执行的，因为新生代产生无法接受该算法产生的碎片垃圾。新生代配合ParNew收集器使用</p>
<p><strong>优点</strong>：并发收集，低停顿</p>
<p><strong>不足</strong>：</p>
<ul>
<li>无法处理浮动垃圾，并发收集会造成内存碎片过多</li>
<li>由于并发标记和并发清理阶段都是并发执行，所以会额外消耗CPU资源</li>
</ul>
<h2 id="CMS相关JVM参数"><a href="#CMS相关JVM参数" class="headerlink" title="CMS相关JVM参数"></a>CMS相关JVM参数</h2><p>-XmxM4096M：设置JVM最大可用内存为4096M。<br>-Xms4096M ：设置JVM初始内存为4096M。此值可以设置与-Xmx相同，以避免每次垃圾回收完成后JVM重新分配内存。</p>
<p>-Xmn1536M -Xss256K -XX:+DisableExplicitGC -XX:SurvivorRatio=2 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=0 -XX:+CMSClassUnloadingEnabled -XX:LargePageSizeInBytes=128M -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+PrintClassHistogram -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -Xloggc:/export/Logs/gc.log -XX:HeapDumpPath=/export/Logs/gc.hprof -Dfastjson.parser.autoTypeSupport=true</p>
<figure class="highlight plain"><table><tr><td class="code"><pre><span class="line">-XX:+UseSerialGC：在新生代和老年代使用串行收集器</span><br><span class="line">-XX:SurvivorRatio：设置eden区大小和survivior区大小的比例</span><br><span class="line">-XX:NewRatio:新生代和老年代的比</span><br><span class="line">-XX:+UseParNewGC：在新生代使用并行收集器</span><br><span class="line">-XX:+UseParallelGC ：新生代使用并行回收收集器</span><br><span class="line">-XX:+UseParallelOldGC：老年代使用并行回收收集器</span><br><span class="line">-XX:ParallelGCThreads：设置用于垃圾回收的线程数</span><br><span class="line">-XX:+UseConcMarkSweepGC：新生代使用并行收集器，老年代使用CMS+串行收集器</span><br><span class="line">-XX:ParallelCMSThreads：设定CMS的线程数量</span><br><span class="line">-XX:CMSInitiatingOccupancyFraction：设置CMS收集器在老年代空间被使用多少后触发</span><br><span class="line">-XX:+UseCMSCompactAtFullCollection：设置CMS收集器在完成垃圾收集后是否要进行一次内存碎片的整理</span><br><span class="line">-XX:CMSFullGCsBeforeCompaction：设定进行多少次CMS垃圾回收后，进行一次内存压缩</span><br><span class="line">-XX:+CMSClassUnloadingEnabled：允许对类元数据进行回收</span><br><span class="line">-XX:CMSInitiatingPermOccupancyFraction：当永久区占用率达到这一百分比时，启动CMS回收</span><br><span class="line">-XX:UseCMSInitiatingOccupancyOnly：表示只在到达阀值的时候，才进行CMS回收</span><br></pre></td></tr></table></figure></div><a class="more" href="/box/2aa0380c/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/d107926b/">茶——人在草木间</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2019-12-03</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/杂文/">杂文</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/茶/">茶</a></span><div class="content"><h1 id="茶——人在草木间"><a href="#茶——人在草木间" class="headerlink" title="茶——人在草木间"></a>茶——人在草木间</h1><h2 id="历史起源"><a href="#历史起源" class="headerlink" title="历史起源"></a>历史起源</h2><p>茶之为饮，发乎于神农氏</p>
<p>唐以前不普及，主要是王公贵族饮茶</p>
<p>唐朝发展壮大，陆羽《茶经》</p>
<p>宋茶顶峰时期，斗茶成风</p>
<p>明朝由茶饼到散茶，烹煮到清饮</p>
<h2 id="六大茶类"><a href="#六大茶类" class="headerlink" title="六大茶类"></a>六大茶类</h2><p><img src="https://i.loli.net/2019/12/03/b5l8XGTICz7RNFw.jpg" alt></p>
<p>绿茶：不发酵</p>
<p>白茶：微发酵</p>
<p>黄茶：轻发酵</p>
<p>乌龙茶（青茶）：半发酵</p>
<p>红茶：全发酵</p>
<p>黑茶：后发酵</p>
<h2 id="中国十大名茶"><a href="#中国十大名茶" class="headerlink" title="中国十大名茶"></a>中国十大名茶</h2><p>西湖龙井：“色绿、香郁、味甘、形美”，“狮（峰）、龙（井）、云（栖）、虎（跑）、梅（家坞）”，外观扁平光润、挺直尖削，栗香、豆香</p>
<p>洞庭碧螺春：太湖洞庭山，“条索纤细、卷曲成螺，满身披毫”，茶果间作，花果香，“吓煞人香”</p>
<p>黄山毛峰：“绿中泛黄，色如象牙，鱼叶金黄”</p>
<p>六安瓜片：形似瓜子，叶面向内翻卷，着白霜色，无芽无梗</p>
<p>信阳毛尖：“细、圆、光、直、多白毫、香高、味浓、汤色绿”</p>
<p>君山银针：茶芽金黄，白毫显露，外形象一根根银针，“白鹤茶”，“黄翎毛”，“金镶玉”</p>
<p>都匀毛尖：“干茶绿中带黄，汤色绿中透黄，叶底绿中显黄”的“三绿三黄”</p>
<p>祁门红茶：“祁红特绝群芳最，清誉高香不二门”，“群芳最”、“红茶皇后” 印度大吉岭茶——“茶中香槟” ，锡兰红茶乌瓦茶——“献给世界的礼物”</p>
<p>武夷岩茶：岩韵，岩骨花香，分为肉桂（灌木）、水仙（小乔木），奇种（五大名枞：大红袍、铁罗汉、白鸡冠、水金龟、半天腰）</p>
<p>安溪铁观音：茶叶紧结，叶身沉重，圆整呈蜻蜓头，青蛙腿，“观音韵”</p>
<p><strong>发酵</strong>指人们借助微生物在有氧或无氧条件下的生命活动来制备微生物菌体本身、或者直接代谢产物或次级代谢产物的过程</p>
<p><strong>茶叶的“发酵”</strong>其实是在细胞壁破损后，存在于细胞壁中的氧化酶类促进儿茶素类进行的一系列的氧化过程。这一过程更像是一系列的酶促反应</p>
<p><strong>茶叶的后发酵才是真正意义上的发酵，由微生物参与起作用</strong>，这个过程与发面团、做酸奶、酿酒的发酵是一回事。</p>
<p><strong>后发酵</strong>，说的就是在杀青后进行发酵，这是制作黑茶的必要工序，在制作黑茶的时候，杀青、揉捻之后，人们就把茶叶的半成品堆放在一起，利用湿热促进微生物发酵，这个过程就叫<strong>“渥堆”</strong>。</p>
<p>生普：杀青不杀透，部分酶的活性被保留了下来，长期的保存中缓慢发酵，越陈越香，<strong>酶促发酵</strong></p>
<p>熟普：<strong>微生物后发酵</strong></p>
<h2 id="茶的化学成分"><a href="#茶的化学成分" class="headerlink" title="茶的化学成分"></a>茶的化学成分</h2><h3 id="茶多酚（涩）"><a href="#茶多酚（涩）" class="headerlink" title="茶多酚（涩）"></a><strong>茶多酚</strong>（涩）</h3><p>茶多酚是茶叶中多酚类物质的总称，为白色不定形粉末，易溶于水，其中含有30 种以上的酚类物质。茶多酚按主要化学成分分为儿茶素类、黄酮类、花青素类、酚酸类四大类物质。在这四类物质中，儿茶素含量最高，占茶多酚总量的60%-80%，其次是黄酮类，其他酚类物质含量比较少</p>
<h3 id="生物碱（苦）"><a href="#生物碱（苦）" class="headerlink" title="生物碱（苦）"></a><strong>生物碱</strong>（苦）</h3><p>主要有咖啡碱、可可碱、茶叶碱、腺嘌呤、鸟嘌呤等。其中含量最多的是咖啡碱，占茶叶干重的2%~4%，其次是可可碱，约占总量的0.05%，再其次是茶叶碱，约占0.002%。</p>
<h3 id="糖类物质（甜）"><a href="#糖类物质（甜）" class="headerlink" title="糖类物质（甜）"></a><strong>糖类物质</strong>（甜）</h3><p>茶叶中的糖类物质包括单糖、寡糖、多糖及其少量其他糖类。其含量占干物质总量的20%～25%。</p>
<p>单糖和双糖又称可溶性糖，易溶于水，含量为0.8%～4%，是组成茶叶滋味的物质之一。茶叶中的多糖包括淀粉、纤维素、半纤维素和木质素等物质，含量占茶叶干物质总量的20%以上，多糖不溶于水，是衡量茶叶老嫩度的重要成分。茶叶嫩度低，多糖含量高；嫩度高，多糖含量低。</p>
<h3 id="氨基酸（鲜）"><a href="#氨基酸（鲜）" class="headerlink" title="氨基酸（鲜）"></a><strong>氨基酸</strong>（鲜）</h3><p>氨基酸是组成蛋白质的基本物质，含量占干物质总量的1%～4%。茶叶中的氨基酸已发现有茶氨酸、谷氨酸、天门冬氨酸等26种,鲜爽的口感。</p>
<h3 id="果胶（稠）"><a href="#果胶（稠）" class="headerlink" title="果胶（稠）"></a><strong>果胶</strong>（稠）</h3><p>茶叶中的果胶等物质是糖的代谢产物，含量占干物质总量的4%左右。果胶的存在有利于茶叶加工过程中手工揉捻成形，且跟茶汤粘稠度等有关。水溶性果胶是形成茶汤厚度和外形光泽度的主要成分之一。</p>
<h3 id="色素"><a href="#色素" class="headerlink" title="色素"></a><strong>色素</strong></h3><p>茶叶中的色素包括脂溶性色素和水溶性色素两部分，含量仅占茶叶干物质总量的1%左右。脂溶性色素不溶于水，有叶绿素、叶黄素、胡萝卜素等。水溶性色素有黄酮类物质、花青素及茶多酚氧化产物茶黄素、茶红素和茶褐素等。</p>
<p>随着氧化程度增高，茶黄素—>茶红素—>茶褐素</p>
<p><strong>茶黄素</strong>：明亮，鲜爽，易氧化，“软黄金”</p>
<p><strong>茶红素</strong>：刺激性较弱，醇和</p>
<p><strong>茶褐素</strong>：淡薄</p>
<h3 id="其他"><a href="#其他" class="headerlink" title="其他"></a><strong>其他</strong></h3><p>水分、芳香物质、无机化合物、蛋白质等等</p>
</div><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/ccf7cbc/">Git多仓库配置</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2019-10-17</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><div class="content"><h1 id="Git多仓库配置"><a href="#Git多仓库配置" class="headerlink" title="Git多仓库配置"></a>Git多仓库配置</h1><p>原文见：<a href="https://www.cnblogs.com/bwar/p/9297343.html" target="_blank" rel="noopener">https://www.cnblogs.com/bwar/p/9297343.html</a></p>
<h3 id="远程仓库的git-config"><a href="#远程仓库的git-config" class="headerlink" title="远程仓库的git config"></a>远程仓库的git config</h3><figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="string">[core]</span></span><br><span class="line"> <span class="string">repositoryformatversion</span> <span class="string">=</span> <span class="number">0</span></span><br><span class="line"> <span class="string">filemode</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">bare</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">logallrefupdates</span> <span class="string">=</span> <span class="literal">true</span></span><br><span class="line"> <span class="string">symlinks</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">ignorecase</span> <span class="string">=</span> <span class="literal">true</span></span><br><span class="line"><span class="string">[remote</span> <span class="string">"origin"</span><span class="string">]</span></span><br><span class="line"> <span class="string">url</span> <span class="string">=</span> <span class="attr">https://github.com/wq/project.git</span></span><br><span class="line"> <span class="string">fetch</span> <span class="string">=</span> <span class="string">+refs/heads/*:refs/remotes/origin/*</span></span><br><span class="line"><span class="string">[branch</span> <span class="string">"master"</span><span class="string">]</span></span><br><span class="line"> <span class="string">remote</span> <span class="string">=</span> <span class="string">origin</span></span><br><span class="line"> <span class="string">merge</span> <span class="string">=</span> <span class="string">refs/heads/master</span></span><br></pre></td></tr></table></figure>
<h3 id="用git命令行添加多个远程仓库"><a href="#用git命令行添加多个远程仓库" class="headerlink" title="用git命令行添加多个远程仓库"></a>用git命令行添加多个远程仓库</h3><p>添加一个名为“mirror”的远程仓库：</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git remote add mirror https://git.coding.net/wq/project.git</span><br></pre></td></tr></table></figure>
<p>执行完这条命令后.git/config文件内容变成了：</p>
<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="string">[core]</span></span><br><span class="line"> <span class="string">repositoryformatversion</span> <span class="string">=</span> <span class="number">0</span></span><br><span class="line"> <span class="string">filemode</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">bare</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">logallrefupdates</span> <span class="string">=</span> <span class="literal">true</span></span><br><span class="line"> <span class="string">symlinks</span> <span class="string">=</span> <span class="literal">false</span></span><br><span class="line"> <span class="string">ignorecase</span> <span class="string">=</span> <span class="literal">true</span></span><br><span class="line"><span class="string">[remote</span> <span class="string">"origin"</span><span class="string">]</span></span><br><span class="line"> <span class="string">url</span> <span class="string">=</span> <span class="attr">https://github.com/wq/project.git</span></span><br><span class="line"> <span class="string">fetch</span> <span class="string">=</span> <span class="string">+refs/heads/*:refs/remotes/origin/*</span></span><br><span class="line"><span class="string">[branch</span> <span class="string">"master"</span><span class="string">]</span></span><br><span class="line"> <span class="string">remote</span> <span class="string">=</span> <span class="string">origin</span></span><br><span class="line"> <span class="string">merge</span> <span class="string">=</span> <span class="string">refs/heads/master</span></span><br><span class="line"><span class="string">[remote</span> <span class="string">"mirror"</span><span class="string">]</span></span><br><span class="line"> <span class="string">url</span> <span class="string">=</span> <span class="attr">https://git.coding.net/wq/project.git</span></span><br><span class="line"> <span class="string">fetch</span> <span class="string">=</span> <span class="string">+refs/heads/*:refs/remotes/mirror/*</span></span><br></pre></td></tr></table></figure>
<p>此时已经是一个本地仓库，两个远程仓库。使用下面的命令可以分别从两个远程仓库拉取和推送到两个远程仓库。</p>
<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="string">git</span> <span class="string">pull</span> <span class="string">origin</span> <span class="string">master</span> </span><br><span class="line"><span class="string">git</span> <span class="string">pull</span> <span class="string">mirror</span> <span class="string">master</span></span><br><span class="line"><span class="string">git</span> <span class="string">push</span> <span class="string">origin</span> <span class="string">master</span> </span><br><span class="line"><span class="string">git</span> <span class="string">push</span> <span class="string">mirror</span> <span class="string">master</span></span><br></pre></td></tr></table></figure>
<h3 id="一条命令同时更新多个远程仓库"><a href="#一条命令同时更新多个远程仓库" class="headerlink" title="一条命令同时更新多个远程仓库"></a>一条命令同时更新多个远程仓库</h3><p>直接在origin中添加一个url来实现一个本地仓库多个远程仓库。</p>
<figure class="highlight plain"><table><tr><td class="code"><pre><span class="line">git remote set-url --add origin https://coding.com/wq/project.git</span><br></pre></td></tr></table></figure></div><a class="more" href="/box/ccf7cbc/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/2c3ea3f5/">Spark——Structured streaming + hive sink</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2019-09-18</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/scala/">scala</a><span class="article-meta__link">-</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/spark/">spark</a></span><div class="content"><h1 id="Spark——Structured-streaming-hive-sink"><a href="#Spark——Structured-streaming-hive-sink" class="headerlink" title="Spark——Structured streaming + hive sink"></a>Spark——Structured streaming + hive sink</h1><h2 id="背景"><a href="#背景" class="headerlink" title="背景"></a>背景</h2><p>需求：Structured streaming消费kafka并将数据写入Hive表</p>
<p>spark 版本 2.3.1</p>
<p>scala 版本 2.11.8</p>
<p>spark从2.4版本以后，支持foreachBatch</p>
<figure class="highlight scala"><table><tr><td class="code"><pre><span class="line">streamingDF.writeStream.foreachBatch { (batchDF: <span class="type">DataFrame</span>, batchId: <span class="type">Long</span>) =></span><br><span class="line"> <span class="comment">// Transform and write batchDF </span></span><br><span class="line">}.start()</span><br></pre></td></tr></table></figure>
<p>所以本次需要自定义sink实现写Hive的操作</p>
<h2 id="Structured-streaming-消费kafka"><a href="#Structured-streaming-消费kafka" class="headerlink" title="Structured streaming 消费kafka"></a>Structured streaming 消费kafka</h2><h3 id="消费kafka数据，对json数据进行transform"><a href="#消费kafka数据，对json数据进行transform" class="headerlink" title="消费kafka数据，对json数据进行transform"></a>消费kafka数据，对json数据进行transform</h3><figure class="highlight scala"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> java.sql.<span class="type">Timestamp</span></span><br><span class="line"><span class="keyword">import</span> com.alibaba.fastjson.{<span class="type">JSON</span>, <span class="type">JSONObject</span>}</span><br><span class="line"><span class="keyword">import</span> org.apache.spark.<span class="type">SparkConf</span></span><br><span class="line"><span class="keyword">import</span> org.apache.spark.sql.<span class="type">SparkSession</span></span><br><span class="line"><span class="keyword">import</span> org.apache.spark.sql.streaming.<span class="type">Trigger</span></span><br><span class="line"><span class="keyword">import</span> org.slf4j.<span class="type">LoggerFactory</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">object</span> <span class="title">StructuredReceiver</span> </span>{</span><br><span class="line"> <span class="keyword">val</span> <span class="type">LOGGER</span> = <span class="type">LoggerFactory</span>.getLogger(<span class="type">StructuredReceiver</span>.getClass)</span><br><span class="line"></span><br><span class="line"> <span class="function"><span class="keyword">def</span> <span class="title">main</span></span>(args: <span class="type">Array</span>[<span class="type">String</span>]): <span class="type">Unit</span> = {</span><br><span class="line"> <span class="keyword">var</span> suffix = <span class="string">""</span></span><br><span class="line"> <span class="keyword">if</span> (args != <span class="literal">null</span> && args.length >= <span class="number">1</span>) {</span><br><span class="line"> suffix = args(<span class="number">0</span>)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="keyword">val</span> checkpointDirectory = <span class="string">"/user/checkpoint/StructuredReceiver"</span> + suffix</span><br><span class="line"></span><br><span class="line"> <span class="keyword">val</span> conf = <span class="keyword">new</span> <span class="type">SparkConf</span>().setAppName(<span class="string">"StructuredReceiver"</span>)</span><br><span class="line"></span><br><span class="line"> <span class="keyword">val</span> ss = <span class="type">SparkSession</span>.builder().enableHiveSupport().config(conf).getOrCreate()</span><br><span class="line"></span><br><span class="line"> <span class="keyword">import</span> ss.implicits._</span><br><span class="line"></span><br><span class="line"> <span class="keyword">val</span> lines = ss</span><br><span class="line"> .readStream</span><br><span class="line"> .format(<span class="string">"kafka"</span>)</span><br><span class="line"> .option(<span class="string">"kafka.bootstrap.servers"</span>, <span class="string">"host1:port1,host2:port2"</span>)</span><br><span class="line"> .option(<span class="string">"subscribe"</span>, <span class="string">"topic"</span>)</span><br><span class="line"> .load()</span><br><span class="line"> .selectExpr(<span class="string">"CAST(key AS STRING)"</span>, <span class="string">"CAST(value AS STRING)"</span>, <span class="string">"CAST(timestamp AS TIMESTAMP)"</span>)</span><br><span class="line"> .as[(<span class="type">String</span>, <span class="type">String</span>, <span class="type">Timestamp</span>)]</span><br><span class="line"></span><br><span class="line"> <span class="comment">//数据transform</span></span><br><span class="line"> <span class="keyword">val</span> resultDf = lines.map(record => {</span><br><span class="line"> <span class="keyword">var</span> json: <span class="type">JSONObject</span> = <span class="literal">null</span></span><br><span class="line"> <span class="keyword">try</span> {</span><br><span class="line"> json = <span class="type">JSON</span>.parseObject(record._2)</span><br><span class="line"> } <span class="keyword">catch</span> {</span><br><span class="line"> <span class="keyword">case</span> e: <span class="type">Exception</span> => {</span><br><span class="line"> <span class="type">LOGGER</span>.warn(<span class="string">"parse josn erro"</span>, e)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="keyword">if</span> (json == <span class="literal">null</span> || json.isEmpty) {</span><br><span class="line"> (<span class="literal">null</span>, <span class="literal">null</span>)</span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="keyword">val</span> appName = json.getString(<span class="string">"appName"</span>)</span><br><span class="line"> <span class="keyword">val</span> msg = json.getString(<span class="string">"msg"</span>)</span><br><span class="line"> (appName, msg)</span><br><span class="line"> }</span><br><span class="line"> }).filter(pair => pair._1 != <span class="literal">null</span> && pair._1 != <span class="string">""</span> && pair._2 != <span class="literal">null</span> && pair._2 != <span class="string">""</span>).map(pair => {</span><br><span class="line"> <span class="keyword">val</span> appName = pair._1</span><br><span class="line"> <span class="keyword">val</span> msg = pair._2</span><br><span class="line"> <span class="keyword">val</span> values = msg.split(<span class="string">"\t"</span>)</span><br><span class="line"> (values(<span class="number">0</span>), values(<span class="number">1</span>).toLong, values(<span class="number">2</span>).toInt, values(<span class="number">3</span>))</span><br><span class="line"> }).toDF()</span><br><span class="line"></span><br><span class="line"> resultDf.writeStream</span><br><span class="line"> .outputMode(<span class="string">"append"</span>)</span><br><span class="line"> .trigger(<span class="type">Trigger</span>.<span class="type">ProcessingTime</span>(<span class="string">"10 seconds"</span>))<span class="comment">//批次时间</span></span><br><span class="line"> .format(<span class="string">"com.warmqing.spark.HiveSinkProvider"</span>)<span class="comment">//自定义HiveSinkProvider</span></span><br><span class="line"> .option(<span class="string">"checkpointLocation"</span>, checkpointDirectory)</span><br><span class="line"> .start()</span><br><span class="line"> .awaitTermination()</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div><a class="more" href="/box/2c3ea3f5/#more">Read more</a><hr></div><div class="recent-post-item article-container"><a class="article-title" href="/box/8be2bca3/">Elasticsearch 深分页问题</a><time class="post-meta__date"><i class="fa fa-calendar" aria-hidden="true"></i> 2019-04-28</time><span class="article-meta"><span class="article-meta__separator">|</span><i class="fa fa-inbox article-meta__icon" aria-hidden="true"></i><a class="article-meta__categories" href="/categories/日志/">日志</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fa fa-tag article-meta__icon" aria-hidden="true"></i><a class="article-meta__tags" href="/tags/Elasticsearch/">Elasticsearch</a></span><div class="content"><h1 id="Elasticsearch-深分页问题"><a href="#Elasticsearch-深分页问题" class="headerlink" title="Elasticsearch 深分页问题"></a>Elasticsearch 深分页问题</h1><h1 id="——from-size、search-after、scroll"><a href="#——from-size、search-after、scroll" class="headerlink" title="——from+size、search after、scroll"></a>——from+size、search after、scroll</h1><h2 id="背景"><a href="#背景" class="headerlink" title="背景"></a>背景</h2><p>订单查询从mysql迁移至Es，分页查询订单有超过1w条的需求</p>
<h2 id="Es几种分页方式"><a href="#Es几种分页方式" class="headerlink" title="Es几种分页方式"></a>Es几种分页方式</h2><h3 id="from-size"><a href="#from-size" class="headerlink" title="from+size"></a>from+size</h3><p>1、coordinate node向index的其余的shards 发送同样的请求，请求查询from+size条记录</p>
<p>2、汇总<code>（shards * （from + size））</code>条记录到coordinate node</p>
<p>3、coordinate node排序记录，最终抽取出真正的 from 后的 size 条结果</p>
<p>索引非常大时（千万级或亿级），无法用这个方法做深度分页，有OOM的风险</p>
<p>Java API：6.3</p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line">SearchResponse response = client.prepareSearch(<span class="string">"index1"</span>, <span class="string">"index2"</span>)</span><br><span class="line"> .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)</span><br><span class="line"> .setQuery(QueryBuilders.termQuery(<span class="string">"multi"</span>, <span class="string">"test"</span>)) <span class="comment">// Query</span></span><br><span class="line"> .setPostFilter(QueryBuilders.rangeQuery(<span class="string">"age"</span>).from(<span class="number">12</span>).to(<span class="number">18</span>)) <span class="comment">// Filter</span></span><br><span class="line"> .setFrom(<span class="number">0</span>).setSize(<span class="number">60</span>).setExplain(<span class="keyword">true</span>)</span><br><span class="line"> .get();</span><br></pre></td></tr></table></figure>
<h2 id="search-after"><a href="#search-after" class="headerlink" title="search after"></a>search after</h2><p>1、search after需要指定排序</p>
<p>官方文档建议：</p>
<p>每个文档具有一个唯一值的字段应该用作排序。否则，具有相同排序值的文档的排序顺序将无法识别。建议的方法是使用每个文档唯一值字段<code>_id</code>排序。</p>
<figure class="highlight js"><table><tr><td class="code"><pre><span class="line">GET twitter/_search</span><br><span class="line">{</span><br><span class="line"> <span class="string">"size"</span>: <span class="number">10</span>,</span><br><span class="line"> <span class="string">"query"</span>: {</span><br><span class="line"> <span class="string">"match"</span> : {</span><br><span class="line"> <span class="string">"title"</span> : <span class="string">"elasticsearch"</span></span><br><span class="line"> }</span><br><span class="line"> },</span><br><span class="line"> <span class="string">"search_after"</span>: [<span class="number">1463538857</span>, <span class="string">"654323"</span>],</span><br><span class="line"> <span class="string">"sort"</span>: [</span><br><span class="line"> {<span class="string">"date"</span>: <span class="string">"asc"</span>},</span><br><span class="line"> {<span class="string">"_id"</span>: <span class="string">"desc"</span>}</span><br><span class="line"> ]</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p>当时使用文档的<code>_id</code>排序感觉很慢，后使用业务中的<code>id</code>字段</p>
<p>2、必须从第一页开始搜起（你可以随便指定一个坐标让它返回结果，只是你不知道会在全量结果的何处）</p>
<p>3、从第一页开始以后每次都带上<code>search_after</code> 排序值，从而为无状态实现一个状态，把每次固定的from + size偏移变成一个确定值，而查询则从这个偏移量开始获取size个doc，每个shard 获取size个，coordinate node最后汇总 shards*size 个。<strong>search after是一个常量查询延迟和开销</strong></p>
<p>Java API：6.3</p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line">SearchResponse response = client.prepareSearch(<span class="string">"index1"</span>, <span class="string">"index2"</span>)</span><br><span class="line"> .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)</span><br><span class="line"> .setQuery(QueryBuilders.termQuery(<span class="string">"multi"</span>, <span class="string">"test"</span>)) <span class="comment">// Query</span></span><br><span class="line"> .searchAfter(sortValues) <span class="comment">//sortValues</span></span><br><span class="line"> .setPostFilter(QueryBuilders.rangeQuery(<span class="string">"age"</span>).from(<span class="number">12</span>).to(<span class="number">18</span>)) <span class="comment">// Filter</span></span><br><span class="line"> .setFrom(<span class="number">0</span>).setSize(<span class="number">60</span>).setExplain(<span class="keyword">true</span>)</span><br><span class="line"> .get();</span><br></pre></td></tr></table></figure>
<p>使用search after方式，参数<code>from</code>必须设置为0（或-1）<br></p></div><a class="more" href="/box/8be2bca3/#more">Read more</a><hr></div><nav id="pagination"><div class="pagination"><span class="page-number current">1</span><a class="page-number" href="/page/2/">2</a><a class="extend next" rel="next" href="/page/2/"><i class="fa fa-chevron-right"></i></a></div></nav></div></div><footer class="footer-bg" style="background-image: url(https://i.loli.net/2019/04/04/5ca58c8caa281.jpg)"><div class="layout" id="footer"><div class="copyright">©2013 - 2020 By warmqing</div><div class="framework-info"><span>Driven - </span><a href="http://hexo.io"><span>Hexo</span></a><span class="footer-separator">|</span><span>Theme - </span><a href="https://github.com/Molunerfinn/hexo-theme-melody"><span>Melody</span></a></div><div class="busuanzi"><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script><span id="busuanzi_container_site_uv"><i class="fa fa-user"></i><span id="busuanzi_value_site_uv"></span><span></span></span><span class="footer-separator">|</span><span id="busuanzi_container_site_pv"><i class="fa fa-eye"></i><span id="busuanzi_value_site_pv"></span><span></span></span></div></div></footer><i class="fa fa-arrow-up" id="go-up" aria-hidden="true"></i><script src="https://cdn.jsdelivr.net/npm/animejs@latest/anime.min.js"></script><script src="https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js"></script><script src="https://cdn.jsdelivr.net/npm/velocity-animate@latest/velocity.min.js"></script><script src="https://cdn.jsdelivr.net/npm/velocity-ui-pack@latest/velocity.ui.min.js"></script><script src="/js/utils.js?version=1.6.1"></script><script src="/js/fancybox.js?version=1.6.1"></script><script src="/js/sidebar.js?version=1.6.1"></script><script src="/js/copy.js?version=1.6.1"></script><script src="/js/fireworks.js?version=1.6.1"></script><script src="/js/transition.js?version=1.6.1"></script><script src="/js/scroll.js?version=1.6.1"></script><script src="/js/head.js?version=1.6.1"></script><script src="/js/search/algolia.js"></script><script id="ribbon" src="/js/canvas-ribbon.js" size="150" alpha="0.6" zindex="-1"></script><script>if(/Android|webOS|iPhone|iPod|BlackBerry/i.test(navigator.userAgent)) {
$('#nav').addClass('is-mobile')
$('footer').addClass('is-mobile')
}</script><div class="search-dialog" id="algolia-search"><div class="search-dialog__title" id="algolia-search-title">Algolia</div><div id="algolia-input-panel"><div id="algolia-search-input"></div></div><hr><div id="algolia-search-results"><div id="algolia-hits"></div><div id="algolia-pagination"></div><div id="algolia-stats"></div></div><span class="search-close-button"><i class="fa fa-times"></i></span></div><div class="search-mask"></div></body></html>