From 10b7e59d9a754dddbf72b1eb93fb0dcd5b29c799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9as=20Hanss?= Date: Fri, 9 Jun 2017 11:41:15 +0200 Subject: [PATCH 1/7] Updating bootstrap file in order to dispatch logs to graylog instance --- images/apache_php54/Dockerfile | 3 ++- images/apache_php54/bootstrap.sh | 3 +++ images/apache_php56/Dockerfile | 3 ++- images/apache_php56/bootstrap.sh | 3 +++ images/apache_php7/Dockerfile | 3 ++- images/apache_php7/bootstrap.sh | 3 +++ images/solr4/Dockerfile | 3 ++- images/solr4/bootstrap.sh | 3 +++ images/varnish/Dockerfile | 3 ++- images/varnish/bootstrap.sh | 3 +++ 10 files changed, 25 insertions(+), 5 deletions(-) diff --git a/images/apache_php54/Dockerfile b/images/apache_php54/Dockerfile index b1e608c..3c4e210 100644 --- a/images/apache_php54/Dockerfile +++ b/images/apache_php54/Dockerfile @@ -24,7 +24,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless; + openjdk-7-jre-headless \ + netcat; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php54/bootstrap.sh b/images/apache_php54/bootstrap.sh index 53b4998..69b0493 100644 --- a/images/apache_php54/bootstrap.sh +++ b/images/apache_php54/bootstrap.sh @@ -55,6 +55,9 @@ trap clean_up SIGTERM service apache2 restart +# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) +tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & +tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & echo [`date`] Bootstrap finished diff --git a/images/apache_php56/Dockerfile b/images/apache_php56/Dockerfile index 2c761ad..627b5ce 100644 --- a/images/apache_php56/Dockerfile +++ b/images/apache_php56/Dockerfile @@ -24,7 +24,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless; + openjdk-7-jre-headless \ + netcat; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php56/bootstrap.sh b/images/apache_php56/bootstrap.sh index 8307a73..598176c 100644 --- a/images/apache_php56/bootstrap.sh +++ b/images/apache_php56/bootstrap.sh @@ -56,6 +56,9 @@ trap clean_up SIGTERM service apache2 restart +# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) +tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & +tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & echo [`date`] Bootstrap finished diff --git a/images/apache_php7/Dockerfile b/images/apache_php7/Dockerfile index bea14ab..5069150 100644 --- a/images/apache_php7/Dockerfile +++ b/images/apache_php7/Dockerfile @@ -25,7 +25,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless; + openjdk-7-jre-headless \ + netcat; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php7/bootstrap.sh b/images/apache_php7/bootstrap.sh index 8307a73..598176c 100644 --- a/images/apache_php7/bootstrap.sh +++ b/images/apache_php7/bootstrap.sh @@ -56,6 +56,9 @@ trap clean_up SIGTERM service apache2 restart +# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) +tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & +tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & echo [`date`] Bootstrap finished diff --git a/images/solr4/Dockerfile b/images/solr4/Dockerfile index 5fc2e17..59aa008 100644 --- a/images/solr4/Dockerfile +++ b/images/solr4/Dockerfile @@ -13,7 +13,8 @@ RUN echo $DOCKER_TIMEZONE > /etc/timezone; dpkg-reconfigure -f noninteractive tz # Base packages # ----------------------------------------------------------------------------- RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -qqy \ - openjdk-7-jre-headless + openjdk-7-jre-headless \ + netcat; COPY ./solr_4.10 /opt/solr COPY init/solr /etc/init.d/solr diff --git a/images/solr4/bootstrap.sh b/images/solr4/bootstrap.sh index 85d1a78..cfe36fe 100755 --- a/images/solr4/bootstrap.sh +++ b/images/solr4/bootstrap.sh @@ -15,6 +15,9 @@ trap clean_up SIGTERM service solr start +# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) +tail -f /opt/solr/logs/solr.log | nc -w 1 -u graylog-server 5556 & + echo [`date`] Bootstrap finished tail -f /dev/null & diff --git a/images/varnish/Dockerfile b/images/varnish/Dockerfile index bc7c489..94bba76 100644 --- a/images/varnish/Dockerfile +++ b/images/varnish/Dockerfile @@ -19,7 +19,8 @@ RUN apt-get update && \ vim \ htop \ procps \ - net-tools; + net-tools \ + netcat; # varnish # ----------------------------------------------------------------------------- diff --git a/images/varnish/bootstrap.sh b/images/varnish/bootstrap.sh index 00e0f34..f341b8b 100755 --- a/images/varnish/bootstrap.sh +++ b/images/varnish/bootstrap.sh @@ -21,6 +21,9 @@ service varnishncsa start varnish-agent +# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) +/usr/bin/varnishncsa -a -P /var/run/varnishncsa/varnishncsa.pid -F "{\"method\": \"%m\", \"X-Match\": \"%{X-Match}i\", \"X-Location-Id\": \"%{X-Location-Id}i\"}" | grep -P --line-buffered "(BAN|PURGE)" | nc -w 1 -u graylog-server 5555 & + echo [`date`] Bootstrap finished tail -f /dev/null & From 1192e5e030d55f7514e3a77b72f68852f1286622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9as=20Hanss?= Date: Tue, 13 Jun 2017 12:41:15 +0200 Subject: [PATCH 2/7] Add support for graylog in docker stack --- Readme.md | 8 +++ data/graylog/dump.graylog.gz | Bin 0 -> 14702 bytes data/graylog/inject-graylog-config.sh | 1 + docker-compose-dev-template.yml | 25 +++++++- docs/graylog.md | 88 ++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 data/graylog/dump.graylog.gz create mode 100644 data/graylog/inject-graylog-config.sh create mode 100644 docs/graylog.md diff --git a/Readme.md b/Readme.md index 80d76be..a7bdf90 100644 --- a/Readme.md +++ b/Readme.md @@ -261,6 +261,14 @@ You can also mount your own bash aliases by mounting a .bash_aliases file in /ho Solr admin interface can be accessed either through port 8983, i.e http://localhost:8983, or with the /solr/ url, i.e http://www.mysite.ezdev/solr/ . +### Check containers logs + +You can various logs for debugging through Graylog stacks. + +Just go to http://127.0.0.1:9000/ and fill the form with admin/admin. + +For further explanations see [this documentation](./docs/graylog.md) + ## Stopping the Environment ./stack.sh down diff --git a/data/graylog/dump.graylog.gz b/data/graylog/dump.graylog.gz new file mode 100644 index 0000000000000000000000000000000000000000..c668b41d8282e27b717af105f006d7032f87f7a6 GIT binary patch literal 14702 zcmYkjbyOTr&@Q~VI{_9CZUKTj!QB@P7Tn!ExVyW%y9N!i5Zra~;10nrzxVs@d+s^= zN6mEgOx4rXv)xZslSLr{07!P>fcMiLPajIDj7x)5fw8?@Don%<+(FjG3F#s`lGGFv z+?bjSmBmzIncT4N)43%onVxxXQ!FMKvWs^$k6xtE;wOxX^&s zy%N=56yx>9PM#IY0iNo~vd9Fm0G`zR>Tp%0_e)P8L^;13)>Kj04-e3Hf#w0zS0Jm} z&*R4h_(nt9*HcdZ8G#E1fM?=YLjDC1ri~+1Th6u@h3tS^07mHs2Ym+sM#75O@XY9( zd|VIXwXh*q|H=kg)^ZDrWkgHSj0g6Gg?!X6zo zu`1^(!zc<;u}T=gn%awzmjl^7OAlXJM1m~F=B zAEO!z|AissNp9j?U3x8fh<|O=++fT6iQN{$V!fxFrACT7S3_wP0ASjHieiN1WU37o zlSnX86c%UyCmRQ>R<~u3Qj=%_)G$MnR7f?sQEO;i*cg!>lHUYG21@-%8ZXda3)PUQ z@z}DXASJwJL{7F`*<3eXad^DLW0V`c{hM`_@N_VJ-o|iXWmJ^5wq{{V_Ih~NHnobr z=DgN?Hn>USGYv^$0pWfE6YB<2kC$J1#8qf9HUejVQ-DCfv`8aa`NP1CRG@3+Vc~04 znF`zlD6MS^ZU@vdgUPSij*fzDI#|`scJ@E%#gA>5oxV`b+b;<(azlp-fy~E&q|B( z{G~j4^+*|2SSWjq{QJB=cWUUQs_KE%16%KVw%@efq^)Lo%I}-3`u4_DxUW-@x`#5} zB!@LB)))t7)~R+9w{Li%Y26^zA2R!3pEpFb{np$r-O5}=SiSfXO5F^p0trh&xR!Y} z#=j;wMO}U2aHhXZ^h8{0$E}bnN;j)VwVk{-o{(bI2#3sKkSF@#8iI@Lv>K7arW{Za zgkyuf9zckrwFJ-~a3)&2A|N-a5xX7^}(YuYqgew%?zit$4TllV`6H*FpYoWM_FMvs5G z@mGDAH~}^exxg_oR*MM>Kv3{qiyxzdVM2$`Pfgr<6ae|y=ygM8wNSqI$msh>eG>4y zcnsapOiZ7)WZE^fez}koL>=+|B$K>YdN;MiF+U$#eqL5&Rs9~E7`c(9sbZ=hfk*`diZ*~(BQ>?>ugaRPY-RI3L`$rqSSFvaN{x;0}oA`=iT|$S?fMQgkksb^H zjIJpA1{6Nc3FoYU3|84IZ7AtWKhY_PbkOa%cE~u2ETR%DAooS0Z5yZt=b_jM6e0Eg zCVzQmMxtH40%|uPK3pQ>67r!ET~Y?&{-c~CxMUH^fUxg0&_L{Ysz_h7asMghz(d^U zE2E($E`+rC*h=7{?x!*Ap$(M-;L1h&qYbzVAcpm-k~JbEX52rpW>zY++=Lxid5|;Z zsb?_oc%T3)-+vCqk!i+_G+xPbaq<2xX?)(bgdjo&pPN=Ht~~g1WS##aLlAdp(f+Z% zkwB;EoX3)d(>CE$`1XAbTrxzbN!+an${1TFa0u@2c@kto533RvL?(EgnUiQnu!tb463;0ai|yX+`HW)rQ#V6ki+~n zR9xytk%S!PgZj}RAkWJyh(ugXMkZa_m{GB;Ma;7dVCMenN)AhGqx_BqK<5f(RFa26 z;1Ar&?OR@ErODt;&X7J^%LAMI45DsKpt*dOkJ+8;+cv*V`U<&hY9|@{6WZOk{jiHP z?85gGH!rfpIH_+N1EPg1Ru=y(Q?zS6<_f#3MZumrK=C9cUo&DH7BEOQybxPLL#sBn z46?ED&&K79?9aGQ+~gk))yCt29s)g-l_p$$T?V;C{^f{@qf#3)@gZxC4|l;??jOhz`MswzEF%A!2^BF%G?GaP>?P6`;&zV>*jg3B)!Dy5JZmzMSb=g0Dlg z8(g@u64B7l9j2>J*K%3zMxZY-l^Zl$uutB;7x zk)|-f&F_wgV>`o|eP?9^^kx4k_vPaB&}DNtMu@YMJgfX_MRX+X)wwGbkHgZppF z3NlsF=f8uB@a_cjqSOgXW(E6eH0>XpRxtHK*0(U4m@W8yW3J?>BD%Z_o+kRnK0bvnc5~4_!EAP$rlbdV| z=#bFfGyU0p)g=+SADDjlsax@Ap4KjVF_ICGqm@_E@kPQf1E;uqAf;epdjN3B3e#sv zkVGG96+ZJFXGk=e;Wu$IuO+F>=Ys<1+z^XwV^3<#(*@rzP~&O;2EbqGk{;SXzo;%7 z^YcJI8gwD*;rJ_#fTD;KEIEtV5&S+prxK9H??R90`68r2dWV8={3HYbAj_5+S3G3u zQ$C>Yps2CDu=lDZFow1(;8-ax7yh+^Na>z!FiDSlI)@g8gN^bPPRCyId5}T_W*A>J znuW}#l=YW)4*^lG+Kp4_p{})Vd$^G<4;sP4NXT2eLuoY1RLhk%{@hLYT)yz56_K*{ ztHgyQ@Op03vf$Ex+QUG!pN+@oABjQUt1uN)0GBRv0p+_wS*{%SuNM|ngEx<~Ujoam zEVP^G0EaZRJvsW(bs+kqJ=*94PgjtDCZmLkKSQ|(M!gWt5G&JoG;E*{8NM2>V?ptp zMq}cla5O@M{%+mZ3aw!!QUIVA#>`ltl5@DJeE8p`y| zljhF3(x16voYU`ogG7G?C1GCu^|+?{@d)WgVV47@Mt(hX zI_{_#14Yh$VM_%E>IGP(Y=1NE{;n7rR*s{3q|z)dsBzegc>npsU78Z=^6y<{FXGbA zlKC*MJy^Acgxj+yioTS$Fn50Wn;7#-M)`{TAybxmyH;TOiH97~E&AJ>fng`)D409N zZNjI5VM_>ZC(_b6$3P1a^Iv9N6mC}quW39ZE&u7mUA7b-QqclC>{@3xm27{9P-iaM z?YfIZ=O&qd;c-8FB!_oH4^@+J=I#me)=LdpeloQL>DPfnAser0>fvvCr@S z^asA|8X*QuA+eu5ww2ot%6!OQ_n9mn7+@tMY*5WuQ!4L1xM>cW66|THSm*jy-hGdT zn{pAVnEgMh2cWl5<&G1{Cy6Npc>S(|3I}{^kdUm(<5RVBklIR($|IHpwa*&_zRW2} za(@XoWrV_)AsFnh0^-tJDibY#3Ek{1QIaUS{vHX=d#JDg`AMlfe#?uL+<&+oT`wzC zd9;rNM+kk20h7@&oS$&>r9300EPYW1X^?F+_g4tyt0d4d_}>Q!eG)B^SMikjuP1sW zc8jp+r=Lck$!VydP%Td|aDmY%W}d{bv^P zYsZ6RnbSYy81U{b(RN#2Pk#lfB`CMJ{N*I)f9zHTwRSfug(yD)B}|+Dvl~}}dh0(k zc_sF1|1*yFhvgZ>^)ynW~(zh-;>`sgbvZ0_g# zp%CIX`2q-ZDuUIcp>BuR(9rq`R?do%I|j=9)j)c^uZ5EGsPZcmyk!mAloO`* zK9w(>o!7`z8zUnJZ35^SVx9BuK5lg*>|acmzracS{=V`TWD~pN91j8u0l8d3P1iubB^2 z3fy6NE-dnp3$DE)W&dZG({NL!?miy=*M19ToG2UF1SzGrP_rp7y}M8ePK{m##4Kwk z&d{a?+ALRjNJ8| za9xWjLG8SyK$I2fLtWrRmZRu)my|@51>PW|0+AMSE4{hx6rd^Ye~v+xQ|WkZ`G}jy zyu#~-4}fF?{F)v=!bFr>TII)oK;4cCtbO1FEGYl*|05!%GA=%Xhjg(#diF~81AA)G zUg5jWj}H5Mk>JVyAil#*f%y-6B02_^^AGG#OUt`^Ke}R4|Asd@IavTunQl)i$y z$d1(dLBC{GJoiJ(5*N#T(h9F{DgO)N2ReqqypN$a<6sajD1VtEyRE=vco(^FY1*#X z<7BvHhH*SJe*&?7dh&h(hnA^+cvYH5a=xDtc(&#hbE>$0l0`F9(Y6;L|B!J`Y<{*$pw__`Vbar1uF_x=|=;s_V4R9 zoYyP|&o07MBixd~;Uf@I{hy-j7Hve+UOI2o;pr5lLFHMS;rvx83wvZ;X8UbF?h4Rd z#v*?&YHiE_tIYPZ9s^})B(qgLB7t=h_lf5TaP%g5Gx37U^3+`>oyKr7SaK*aPDiV^ zW4WolX0`e#%~J34oXz-4V$TC+g~??a75}Xx<6l&fWQ2Yn#>q=}=l7Oc1&W9*kw2jz zTLg>YJDUpmi$g8NPeW7%IAo;+O!d_ZbXB|C7*&x1M1JWqho5ckPp}NUV3nwX`fUHo zj&rVEaef5=2AgguyIBZG@AjdKer)2KNx?2&;0%b;Asl-Cy@`PzDDpHI%?n~e4Gnp* z2zYo7+SVn%lpoW|OARdjvOL^5qS4&n8e;Lus#raN+?1TWidO|Ky9ta{J6geU30?t(5`&XsTZ;3(wPbWDkQK<`Y8@NK&@i55k!w&#q@?Bv81b0D_ubZj7K5 z>JB8dFmX=}NQ4|d=*0vECsJqxF{eFztytpiOL06NEt}I1HRv$1Z2R^8ZLt>Lr?QQQ zBeK9Jq1Qkf-_=n9Vf4dCS&`rR955&A<^)szW1J>J{q)NF6P_OFcl$7+=|EFw3(`r% zH!Op8jj@kqsKzyAa$?>>GBJ%T{F+ANJ!8DW9n9JWj9!!gpEAoeB9>>ZOz0!#U_BE% zHsX_yuDr@nNFsArVhUR&>f_F;=Ka~O-F7AN2LoPlF*NAoB z#=7r!vKAzy8M7{5m-;IApwli5Id$0OB4<;6wedp2M)UOG8Ov*0@U zyh&R-^`k$lNY9 z4y?5a|M5Ed$Bf}oAXqf%$9Bbe2|Uc5s(*Eh9oLRA0U6HiS0S=CP2FKU9-0(bEQ8rh zq$lN@`A0X>Pa$T<#nquTtx(Syg+#I8vvq zKkp;=;0%d5jV5za_bTF-jV;4@R?9f8d~{|= z-4OmHH$tPip+Z&FaV{QL;y12E2!;dvOOHu@&ytQPMJvzzpjYf7GvNRe2cbm?z&vw~ zgN~F3;w<*}+X<3~Gt`Ev^N$xTuH7;`+;3~!H}hpVw^84g*#y7AVi76?u!XUG@H^f{ zK-p8Ee0ug64b5(bjg1q7kpR1{>yN~}m(67@BZ>sU=3^(RJ7YxgV)8HXz(198!a^yE zXWwI*sb5oI?q-Fhi-l1DtQoRq4_Sf!vWLDn286Ma4Bg$THJ6S2IvXf6KY(NjV+CW| z7QuBw*LLn8(pbsss0HJ53r-D5;UQ?gDGVt}M)etznoW?%Z;`VLXujCm8m4?%+y#;v zGeb`vLX{t15mZ*_K}L7s{FUfZl-Rtf$*IbpS3m7y8aPeZk~&WvtQy@&W~IJ*55OyE zt7Nei?A;cRg+T#OrRsW9wU+cZ*@eT7^H#Pq#CitXqWO_r-NL#Ajl0$)i~U)>5zl>& z2&fbATeMt4(>tfwpP(;dt=sIY(7WR5&V6QLrna62^ti=qcMF?xS29*QPTZ$#?d7u5 zR>E&&C???m0sl;xzG4xPDUGlL)rzYc?slF3FWsWo<9OoO<6YNEJs@Wz{tp-H2m9wh zFsHTq_9~8A1;@S4Xo0PuPzsnTaYb4?aly@Zk&?K3w@HOk161$sZZi+T{;~kx1M!0Dj7x|l^NBYxN|WT(QSP$WcqZzr zo@_ysN++5v>cAXnT^;JPY%Si+f5pSuHb`Qtggho0IW-?Ev*_petHq#MedENY^11pb z#pQ&H@rJCUduAI*EMD9`+0^}I@g8i>P zxjHUz@)zl0jSp8Cy0}ep2JfmAYnUK`d0h@s`(84u#@+TFBn$DtynvoIX6)o|DxZ}3=Fx}zbPu}J_r&T0>YRZ+hYrIyQ{up!p zxuLd>4>seXX&jI6-HGUr-CiMC-QthmwXHfJTb_UMpLmNnY7}D5RVfV7l`xzXz8_Yx6pK}d% z?W!2FE6ldzRgn;<+M1D>{j-_Z=onms&#Kw>^1Szo#YVV0gM>z2tJ^JE?(_13m(}Tf z^&2F=DeLwj=xtpa2VO}r=mwm|Dksg0d}|(>4a1AuABLY(fnQ~7usz&8lr=YM+||WUizs*g_R7B7 zu!Fo~VglZckfos`Qkj@%YksdNEcP1L;!j(LEy`3j)V5nOv$@sxr8iZ)cO6)zZLk(L zw^10G`^Bj=NYUiBGXJpqS_T;<=26PRG3@$OaFMZzhk(lkg>Yrx0ySo(E{w+-=h}mM zSy%ezWn+Zw1Z?2MZH27fN#x(7o}6(l_-&zf$9s?La;0tZ^(^WH|W2nE|GmehlW(%uK6{*xLlT z9zbSz^_f3U(WT;H{?y`&@S{$AUY|LZC!QQ*d-^i=@jsKRSk$X#>0-qsOdG}Ui+u|J zqgjhD;dfi?Z<=X1@54ink9_3AaAQNnvAkqPkeqVpg8a)j0g}35+TA1H!5L`*UN`Bg zO2qw;Z421cYult3Mk2<4KY{v-L$5T`ISOZLZR7XG37RXTw-+r{;)@~tjb0J465z&W z@|)l40*|a|$`U_Z-D(xuY}$-&#GDmHXo5;H@lTis3BIr0;@LM;Tu2rFk{b}fc5lqivznJ#N+%GeY-sL-dXiT)`H#T% zv0f0*?dO#e)5)R5atk`~;LK?4suc2Ggy>!;k)6+*hTS52+s4Kwk48hnE)u)!SAlj; z!rxKIR|~*6>)NEF)D6BP|8#4~mG38-$YnGmJI?GGR~)kE4a zYQCoJL8&d?tdGS&o0)L~vz)NeCn{tmb8{6#J`UmyLH?7P$#pJ%evY=m9^-O84iHJ~ z`KDyfa;Tz>vu2yEm^r$dIeHO$*v7(O*jltsT`SP*ZSE8$q5=j=?KrGFoN&qqiK|Bn z60*p^zk36VILbgY1lnNV>`?Qz42uJv8a?G@z})N~Dp^3B$3A+=UY zHYHNYEhK%>>K^&jd6KJ0W7mJ>^YMuailBOH@*cT!X0HX$TBGoDmQnYW=na~xS5zK@ zFXbO&dEMs)1-xCt_&ck1_x<_HUQy;8kD4qP4!Xs&B;AFgTEB0fRJo z-G9SN7`bq1R=?2;y5oFbo&6$aR`N>(5E>R%v0uD9F$u_&jICZIsgYbiuRQx*u&^!t z;qRah{%q$$wX&&Mi1tr`uaT3;=O2r*IqUV2*Q)bjP6W-cu#*!S47>G(QJI9#)5`EJ zSsN7=EO6h&*!;!DpwX!K&)3FkbE%G&LucWLr-?cus@B1&e_rFt2{#6~X7WO00ozQ`PWW5jbm^5KNX{?%`m-Uvl5+LW?M759; zpOGBeQl66eqL?WVw6ql7>F;#;-C5CCpaaqzHe^=QmPsctkKGog|2ElqTQ)pTX?zCN z_%j7nHhp$%vtCbQay>hnuWLu`=+!cB!;%kWSJ{Gy~7TAA@8C?N`cmm;qWNt<5$Cj zOEcA`EZm`7EwcQ?;)2>`7WOB~#L_clFi!m@5ujPJ`(Pi~3 zSrQ$hIcIjbu9Buvs;w|&>RDa2G<_%jFYZ8`=c_t5mLvrkVgWg}!td4SjWdmicN5Wt+HpYJqwlh=02T03|kro`-rSduM@Fp;PbMuz=M<7 zc`p~KDu4GCX|8WAKksRz&s2*)4>0T1>gk+efnl20*jQSyhJVzGTCxwVK4iENvDk`N zdbCim@$);co!Yo~uZ=6YuOdP?%#%_(Zqa)X{?2_A5L5|vNc2XBS=qWp+i~$zJkz}X zy*7A#i?ydo>wNSGNKW2hine{dUMqEwdCp1>j?Orx`VHIyuK9*vibUQZupauqwB_+! z`xF379ZWaw2@;DLDksusr%N8?{s#FrQ6?Fb)|*qTTxDAq==`7Up_S zcgzh`l@xWN{Sqq@gxj^NNz1HW z$VrJ!+Qxgzk_+A1gZv7cqC%^;%S!8l|3c`4;vj$Nwo)*SxI_pwU6xTHbHG%W+3d@V zo*JA!Q-y=Jt3sU8*#JHMtI*=Q^dJ1mL&-sBW);Vj;-Xr4uGC4I#Al1M(j9{nP}hh} zb5|SD*}Y4No)=$6waPhT4x?0tx>>Bv=)-JK4@Xtl*6W>qdJ6N}m$Ec%-yY{P(x#d7 zX-iWLCD+>Q#{!Va6OYQdY292(SHD}31TYb!+3G@Pm@|(M$DH8MQ@)H5Rgv( ztsm~LN_Q!4aw_hcH*X4ObWjCjun$YgPTWvXh^z0~GxYA!=jR5q5T_`F?|w*S9~5kAu!xu$J%Kl$ZuFIjUO+bFa~*ss0#_75sbv{uvoWR?ed z9tN08sIy(ZF}*isTeta?Q!gTO|L*V3M!D!cmrh5|_w2o=6VT3LtL>%q<@mE=yNjIn z%S*G?l#;#w;47NWm6VoRH9wo4ACYq86l~xt>COE(+u3b)>OI%(xLBimD6vb9axc;4 z%FUIhGS;s0dtd1Ab1QxHS7fQ14RQNVQev#Tb~yywvJVEW!G}B5`5Ipg22poilH+6g z_|e1-!Ym62D1+ft_WvcezAJ3BTf+R|v-9J2DO8i!R7rmeHSA;v&o@Dg(VFSd9>vK+b|Of;fUd|&wKwp~@Vf&KFcrn;m(aBp@)1$f zr2es?ERSbXQbk#KY2{{xF-LI8Jm$StnHhh{+5y& z^7({>9wE>o&G}3V@8Ia1JLnmEO`Vy$M@D3Bk{lqGc1bBc-quzOeL+_XSJgur@V4OVL#H7qPX9xnHt@GsDf0uuJuZG1qF3v0fRQ zwUs%0%A@5Wc;@`=f^&bg=`=S{;{L3puYc)ps-KfvN8)D_l>8{?v~~zN6)PzU0yI$@-Mk$?~ygdM3~X%z#CqkK$36mVhDy|V5=pi z9Q+fJWLmwde(f+|+Zhv5?7?;XpEl&OF{TRMQrf(>GW_&H*g5_wKJBD2U5i|NNhLxW z2ltWm@>-3}f**X2iJdw;r{=Fk9>c6#1J3j4U4$+dXZ<^iBx@1Pp!iwOILY{hu_p}Fh+)Z3bXoJ$##sR2PT)vS;zd&DY(!CK zXzMI~i6IJY9b z$V^H}>+2;Fb-9PDb8>DpZf3l&}@}ko-z#{T_HgequG>u>O;sOk>z`|)-|b6~s_ zdAjdlI%yllt>9BbP_~~}k*At;Ku6OdQ$^}tT#+bXA3v;4fm$sgj8DJS_!S(OY1XS7>xFJ@ zf|fJq+Rbi`+NmZ{PCh7LJwqyrwTNCwqvk;(q3LJ|z^8wH|GbPd?x_&ZHDNEz+Es=* zyu|c5l&{+kjq}^nOi(F9oxk*rpYxZ>>+mOSofJ8G_2ciRmF6{tGMh2%pZ$&o>z*ge zq%OQ)^>ydZX(T?wHi9f5u|nd$kgNE6nKgtzBxgqH8-^>RH#HD@KVl&W0AMrnSex9< zv@_xFF}*^2L#^JA1XIzady{y&Dpl;f4Ugq`EH9&>JJ=Dar7UPPhQ>7>kIz#9(t_Gk zW2+@631M6K$gNBD^pUYW^vQ_n2Q;ehw4oYCW8l>!Jn<2 zRcvVIx7-!euS^D=1(UTspQ%bPU%{Pe0sKuWF=()U4dH9VihLAZu|SVHROt~8wBZ*p zM0vh0BXUWuUwaKo<&#(TVGRd;YUM7j_8X*QrqayF3v2X?zDn>3@i3f=`?}nwBS)hL z%|n3Kc6=r$G5;KG^wKU1bwE9<`g8>ls$~YNnts%J%s8@&5N)@peb)W?6OOE9b<(x; zY|letVo4iAH}E?0hgf+~n-Q=1?s=qbVbZ4P(;Cc0&&|`)#OCF9UD0Y8q)`z%Wp=?X zX7`4<7rP_-mOAr$o6vDnmC%pm&_{x-DjNvX^PaY9EH?XDp37=amIR&}n6xh5x593r`#Q)qp1X(A4JYkbjO^6)IreGo{z~@Mg`H|Y7_psOh007e#s37N40G&1t!;pSj ze_!E3!JC>MsbFGNX-0&Xjuo{Jot^kwRiEi5 zF{LmtiYA_`3-e4b$>*_izmX#7F#QYb=hWD5BGzGjWl2rnk@MbQ7HjX!NZ1;YsfK?di>AhHsr1DJB2vg-c_hRr<~6w`A?94<6E}}@2}tb zkoue+>6}>5Yi6N7*$n*h^aK)<5PZAN?&|hIYBZ=Qgw7XYHKZY@Of=qw$9TC4TdqAq zcRV2nf(>|kB*Xqq`zp0;em)TUoQk%0ohv{zTBM)wu&4DIf&>dw(C0(WjUg5Iiav9J zx#1rEWy3A}%>T-hIJNcVsD+cdcSGyu%6h`$2Yg$UMVlGr_!>=ypTRaYNCL2X45= zw#IPC_l-UEdw;oi=`2-)R^$RmGNBhQG)wo5!P7VC@b_RjG=2uFxMq|h@~mKWSDahM z+kHr^n~`3Ev+z)~{Kz!NQiF2p&(eu5(#V|Ilx0h z@WvhHnwqKmfQW%IS#%w9x0D~EN`l?ErFZ~4E3{e6vsW%UEDy{Na6?f^xe z0~U|0>lY^hCJhmF?Q{ONCE-Ey1W_h$yXR~|@dUZh+Au1>IKWF)WXK~N;RUeo!Ej?N zrmG0sivI5}%n~PneJB8K@Jo&P6AT2k!Q-f5fdjq|c&)~WQpQ07(YrYWFm_^v(8k;Zgh{MW9cLSJes2izkAB-!PrbOdl8>9h?JJevm)6E%4y6J1kfk%Ot3FZERL?S!lE5D(sKT}vcz zBi%Q}XDY)e)EhZs&!u>^A!AGcDKJ`KYo=eY+6L8-|U@B8JelP1Hrjv&P!B z**UEp)iam9KAkl0F91C8u`F;$yJF*Eo6UdzI~I78!{5cr2iYI3ws+ne>dwi>i3tbh zP^WcXM}>jZB5pOT%EMvd9sKUJHtOLRSxN8sHJ=7Ykhxg zuMVc^vbZRy@4=M=!WbC zTUahyGI3bo*5di?9lZ`wuF{S14_t4!cqMQ`u7sMdN_kNq;AmlQ)zT>UwOu^%sw;cA zSETU)p{yt346gml)c`{UZcFl84Ky~PW@JAM06j;uSM6h;ctk^HU6$pewr0`zY!|qW zsO_m!6r}6R>`7auNdntzf`SieWQHSTH!?Q9k{{l8R2Xi0R@>D%%xAW3aL-oIuhdV5 z%@PVEn@z-!ig49M2M`c$;!kOmsb<)?TJe1+iBlSCa%{6si-9YW?6mdZ@j5$21%k~F z*YzLn@i|(LHQZw6YrK-|<`vYTEF+6WH;3pN6@zX2o!+8Ya zJ2VJ1z~*xZMFB~1={oW~ur?Rd-V{b8=btvbBi`kSU z&8bj1yBlHjD`e8u*ayk8ZwE}88QCSk9Qb|Lr@*g4NVY`+h_9mLnca@vbU}a}uDX*OuD8yo&81+m?%lRbv~DG7(B{U~G970NeGaYeW2`#TI65 zobcM-6kjCgJ)mD05z=`j#m* zJh*vwe$Lz0IF&KE^1}f=hpA5SAV%=&*i>IncdaU~MW50fC9C8p8i7axLKQxy{N3A6 zYYq0&wv4L&&4T}D%Y{fIlHtQo&VhItg<|VLE5>RPlg?`&=gvkXe_5UQlS77wT+QB~ z)lqtprz^L3zg_O7J4AZbFC8oi-KH@WU==p?4p+g$>oT!RA4NH4!!smz#xB1F1Tbp5 z`-bi@Cs#p`b=rv`Q;y4-Q%niY${i}z{USUzqLZZv(-&tJ!&~wlS>JiWVWAeUFg&LA!weh~+A|~mS*k^XHB)pqu5_Z1wVL$>o@?tLU6oIx zYjHTZ_Lru~we4+<@9MAX>;UAcKIzNT)X<-`4sDHj8hXlqQFEEr=bdDqC4UW5fbVs1 zM(Wm4K4`>R?H^s%Zioujp}Jx{!yDjMwY!s$0@v)Ry!A(u^3<+z0->S=ugn;lG!l%? znp10a?kQN3b5xPmGzcEqKixNecmL+B3qKi|o#Pm#jhlnKYZ5^bDhXUaFQXrZcdfns z649LWI|9B_xQai~NX7NftCgaAGLUJTrzZ8<3DTg|Pw+=IVE~O$)5{NHA7#ym`GRAG z)Ek9NJ5$jKnyZ0gU^e48Jqy3ln( z9kl}fYUv2e9BFI9Y;Gjcd8yf36)Z5RNAl7+&kD?{O^CsHo7sXRhA=%o1@^$`=vx|* ztO^OR9R)EMV0&m!D^=EOF<&z5MYpMQI0{@SQLak`=2*c2dUTBi(yUd{`Zub=#7}n!Jcf< zpD8c3zu$ZdoE4BatPINiq_-E3YkKPXB!k9iJ_!&>q{8VlA3TbF9yyF-ii~6!V??7q zul+`Z^u2+0uai*BrU{k^*r*`e-l#!8o@iq{=A~L2(|tv=uWuO5?J+nw4DF^7A0u7^ zTj2u*Mk^dwjI2*6OmWnDrmy*-oP%pjslb>ehTVSCtyMm1-K +graylog logo +
+ + Graylog - Logs Management Stack +
+ + +

Various log aggregator in order to enhance debugging and monitoring

+ +

+ Kaliop logo +

+
+ +This stack is a poke in order to demonstrate the power of such tool. Please follow the guide to accomplish the walkthrough. + +This library is mainly maintained by [Andréas HANSS](https://github.com/ScreamZ), feel free to contact if you are having some questions. + +# Requirements + +- **Kaliop eZ Docker stack.** https://github.com/kaliop/ezdocker-stack +- **Docker & Docker-compose.** Recent version + +# Introduction + +**Why GRAYLOG ?** + +- Open source log management. +- Scale well in large architecture. +- Embedded user authentication system. +- Embedded alert system on various metrics. +- Support for various format, especially well with [GELF](http://docs.graylog.org/en/2.2/pages/gelf.html). +- Awesome admin & UI. +- Support for LDAP. + +**Why not ELK (Elastic Logstash Kibana)** +- Require [Shield/security](https://www.elastic.co/products/x-pack/security) for user authentication handling (Not free). +- Require [Watcher](https://www.elastic.co/products/x-pack/alerting) for user authentication handling (Not free). +- Heavy costly resources consumption and poor performances. +- Logstash has no UI out-of-the-box, require manual configuration. + +# Setup + +## What is logged + +At the moment you're able to receive following logs : + +- **Varnish BAN / PURGE -** While you're not banning using the Varnish agent tool, take note that it works with back-office BANs. + +- **Apache2 Access/Errors** + +- **Nginx Access/Errors** + +- **SOLR** + + +# Troubleshooting + +**Check docker network** +Your containers might be on different network, feel free to update the docker-compose.yml file accordingly in order to match. + +# Future improvements + +**Pre-bootstrap Mongo with config** +Instead of doing the restore, bootstrap the database at compilation with given parameters. + +**Use log aggregator system** + +- ElasticBEAT with graylog sidecar mode or without. http://docs.graylog.org/en/2.2/pages/collector.html (Deprecated) see http://docs.graylog.org/en/2.2/pages/collector_sidecar.html +- FluentD +- Rsylog / Syslog-NG + +**Think about using LDAP system** + +**Optimize index and stream internally** + +**Find a way to use GELF everywhere** + +- In symfony use monolog with specific wrapper that ignore exception in case of not responsing server + +**Format Apache/NGINX logs upstream in config** \ No newline at end of file From 7cd34a83a437d974fb46194c461de9238a7e5a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9as=20Hanss?= Date: Wed, 21 Jun 2017 10:14:28 +0200 Subject: [PATCH 3/7] Move mongo dump to config folder --- .../graylog/inject-graylog-config.sh | 0 data/graylog/dump.graylog.gz | Bin 14702 -> 0 bytes docker-compose-dev-template.yml | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename {data => config}/graylog/inject-graylog-config.sh (100%) delete mode 100644 data/graylog/dump.graylog.gz diff --git a/data/graylog/inject-graylog-config.sh b/config/graylog/inject-graylog-config.sh similarity index 100% rename from data/graylog/inject-graylog-config.sh rename to config/graylog/inject-graylog-config.sh diff --git a/data/graylog/dump.graylog.gz b/data/graylog/dump.graylog.gz deleted file mode 100644 index c668b41d8282e27b717af105f006d7032f87f7a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14702 zcmYkjbyOTr&@Q~VI{_9CZUKTj!QB@P7Tn!ExVyW%y9N!i5Zra~;10nrzxVs@d+s^= zN6mEgOx4rXv)xZslSLr{07!P>fcMiLPajIDj7x)5fw8?@Don%<+(FjG3F#s`lGGFv z+?bjSmBmzIncT4N)43%onVxxXQ!FMKvWs^$k6xtE;wOxX^&s zy%N=56yx>9PM#IY0iNo~vd9Fm0G`zR>Tp%0_e)P8L^;13)>Kj04-e3Hf#w0zS0Jm} z&*R4h_(nt9*HcdZ8G#E1fM?=YLjDC1ri~+1Th6u@h3tS^07mHs2Ym+sM#75O@XY9( zd|VIXwXh*q|H=kg)^ZDrWkgHSj0g6Gg?!X6zo zu`1^(!zc<;u}T=gn%awzmjl^7OAlXJM1m~F=B zAEO!z|AissNp9j?U3x8fh<|O=++fT6iQN{$V!fxFrACT7S3_wP0ASjHieiN1WU37o zlSnX86c%UyCmRQ>R<~u3Qj=%_)G$MnR7f?sQEO;i*cg!>lHUYG21@-%8ZXda3)PUQ z@z}DXASJwJL{7F`*<3eXad^DLW0V`c{hM`_@N_VJ-o|iXWmJ^5wq{{V_Ih~NHnobr z=DgN?Hn>USGYv^$0pWfE6YB<2kC$J1#8qf9HUejVQ-DCfv`8aa`NP1CRG@3+Vc~04 znF`zlD6MS^ZU@vdgUPSij*fzDI#|`scJ@E%#gA>5oxV`b+b;<(azlp-fy~E&q|B( z{G~j4^+*|2SSWjq{QJB=cWUUQs_KE%16%KVw%@efq^)Lo%I}-3`u4_DxUW-@x`#5} zB!@LB)))t7)~R+9w{Li%Y26^zA2R!3pEpFb{np$r-O5}=SiSfXO5F^p0trh&xR!Y} z#=j;wMO}U2aHhXZ^h8{0$E}bnN;j)VwVk{-o{(bI2#3sKkSF@#8iI@Lv>K7arW{Za zgkyuf9zckrwFJ-~a3)&2A|N-a5xX7^}(YuYqgew%?zit$4TllV`6H*FpYoWM_FMvs5G z@mGDAH~}^exxg_oR*MM>Kv3{qiyxzdVM2$`Pfgr<6ae|y=ygM8wNSqI$msh>eG>4y zcnsapOiZ7)WZE^fez}koL>=+|B$K>YdN;MiF+U$#eqL5&Rs9~E7`c(9sbZ=hfk*`diZ*~(BQ>?>ugaRPY-RI3L`$rqSSFvaN{x;0}oA`=iT|$S?fMQgkksb^H zjIJpA1{6Nc3FoYU3|84IZ7AtWKhY_PbkOa%cE~u2ETR%DAooS0Z5yZt=b_jM6e0Eg zCVzQmMxtH40%|uPK3pQ>67r!ET~Y?&{-c~CxMUH^fUxg0&_L{Ysz_h7asMghz(d^U zE2E($E`+rC*h=7{?x!*Ap$(M-;L1h&qYbzVAcpm-k~JbEX52rpW>zY++=Lxid5|;Z zsb?_oc%T3)-+vCqk!i+_G+xPbaq<2xX?)(bgdjo&pPN=Ht~~g1WS##aLlAdp(f+Z% zkwB;EoX3)d(>CE$`1XAbTrxzbN!+an${1TFa0u@2c@kto533RvL?(EgnUiQnu!tb463;0ai|yX+`HW)rQ#V6ki+~n zR9xytk%S!PgZj}RAkWJyh(ugXMkZa_m{GB;Ma;7dVCMenN)AhGqx_BqK<5f(RFa26 z;1Ar&?OR@ErODt;&X7J^%LAMI45DsKpt*dOkJ+8;+cv*V`U<&hY9|@{6WZOk{jiHP z?85gGH!rfpIH_+N1EPg1Ru=y(Q?zS6<_f#3MZumrK=C9cUo&DH7BEOQybxPLL#sBn z46?ED&&K79?9aGQ+~gk))yCt29s)g-l_p$$T?V;C{^f{@qf#3)@gZxC4|l;??jOhz`MswzEF%A!2^BF%G?GaP>?P6`;&zV>*jg3B)!Dy5JZmzMSb=g0Dlg z8(g@u64B7l9j2>J*K%3zMxZY-l^Zl$uutB;7x zk)|-f&F_wgV>`o|eP?9^^kx4k_vPaB&}DNtMu@YMJgfX_MRX+X)wwGbkHgZppF z3NlsF=f8uB@a_cjqSOgXW(E6eH0>XpRxtHK*0(U4m@W8yW3J?>BD%Z_o+kRnK0bvnc5~4_!EAP$rlbdV| z=#bFfGyU0p)g=+SADDjlsax@Ap4KjVF_ICGqm@_E@kPQf1E;uqAf;epdjN3B3e#sv zkVGG96+ZJFXGk=e;Wu$IuO+F>=Ys<1+z^XwV^3<#(*@rzP~&O;2EbqGk{;SXzo;%7 z^YcJI8gwD*;rJ_#fTD;KEIEtV5&S+prxK9H??R90`68r2dWV8={3HYbAj_5+S3G3u zQ$C>Yps2CDu=lDZFow1(;8-ax7yh+^Na>z!FiDSlI)@g8gN^bPPRCyId5}T_W*A>J znuW}#l=YW)4*^lG+Kp4_p{})Vd$^G<4;sP4NXT2eLuoY1RLhk%{@hLYT)yz56_K*{ ztHgyQ@Op03vf$Ex+QUG!pN+@oABjQUt1uN)0GBRv0p+_wS*{%SuNM|ngEx<~Ujoam zEVP^G0EaZRJvsW(bs+kqJ=*94PgjtDCZmLkKSQ|(M!gWt5G&JoG;E*{8NM2>V?ptp zMq}cla5O@M{%+mZ3aw!!QUIVA#>`ltl5@DJeE8p`y| zljhF3(x16voYU`ogG7G?C1GCu^|+?{@d)WgVV47@Mt(hX zI_{_#14Yh$VM_%E>IGP(Y=1NE{;n7rR*s{3q|z)dsBzegc>npsU78Z=^6y<{FXGbA zlKC*MJy^Acgxj+yioTS$Fn50Wn;7#-M)`{TAybxmyH;TOiH97~E&AJ>fng`)D409N zZNjI5VM_>ZC(_b6$3P1a^Iv9N6mC}quW39ZE&u7mUA7b-QqclC>{@3xm27{9P-iaM z?YfIZ=O&qd;c-8FB!_oH4^@+J=I#me)=LdpeloQL>DPfnAser0>fvvCr@S z^asA|8X*QuA+eu5ww2ot%6!OQ_n9mn7+@tMY*5WuQ!4L1xM>cW66|THSm*jy-hGdT zn{pAVnEgMh2cWl5<&G1{Cy6Npc>S(|3I}{^kdUm(<5RVBklIR($|IHpwa*&_zRW2} za(@XoWrV_)AsFnh0^-tJDibY#3Ek{1QIaUS{vHX=d#JDg`AMlfe#?uL+<&+oT`wzC zd9;rNM+kk20h7@&oS$&>r9300EPYW1X^?F+_g4tyt0d4d_}>Q!eG)B^SMikjuP1sW zc8jp+r=Lck$!VydP%Td|aDmY%W}d{bv^P zYsZ6RnbSYy81U{b(RN#2Pk#lfB`CMJ{N*I)f9zHTwRSfug(yD)B}|+Dvl~}}dh0(k zc_sF1|1*yFhvgZ>^)ynW~(zh-;>`sgbvZ0_g# zp%CIX`2q-ZDuUIcp>BuR(9rq`R?do%I|j=9)j)c^uZ5EGsPZcmyk!mAloO`* zK9w(>o!7`z8zUnJZ35^SVx9BuK5lg*>|acmzracS{=V`TWD~pN91j8u0l8d3P1iubB^2 z3fy6NE-dnp3$DE)W&dZG({NL!?miy=*M19ToG2UF1SzGrP_rp7y}M8ePK{m##4Kwk z&d{a?+ALRjNJ8| za9xWjLG8SyK$I2fLtWrRmZRu)my|@51>PW|0+AMSE4{hx6rd^Ye~v+xQ|WkZ`G}jy zyu#~-4}fF?{F)v=!bFr>TII)oK;4cCtbO1FEGYl*|05!%GA=%Xhjg(#diF~81AA)G zUg5jWj}H5Mk>JVyAil#*f%y-6B02_^^AGG#OUt`^Ke}R4|Asd@IavTunQl)i$y z$d1(dLBC{GJoiJ(5*N#T(h9F{DgO)N2ReqqypN$a<6sajD1VtEyRE=vco(^FY1*#X z<7BvHhH*SJe*&?7dh&h(hnA^+cvYH5a=xDtc(&#hbE>$0l0`F9(Y6;L|B!J`Y<{*$pw__`Vbar1uF_x=|=;s_V4R9 zoYyP|&o07MBixd~;Uf@I{hy-j7Hve+UOI2o;pr5lLFHMS;rvx83wvZ;X8UbF?h4Rd z#v*?&YHiE_tIYPZ9s^})B(qgLB7t=h_lf5TaP%g5Gx37U^3+`>oyKr7SaK*aPDiV^ zW4WolX0`e#%~J34oXz-4V$TC+g~??a75}Xx<6l&fWQ2Yn#>q=}=l7Oc1&W9*kw2jz zTLg>YJDUpmi$g8NPeW7%IAo;+O!d_ZbXB|C7*&x1M1JWqho5ckPp}NUV3nwX`fUHo zj&rVEaef5=2AgguyIBZG@AjdKer)2KNx?2&;0%b;Asl-Cy@`PzDDpHI%?n~e4Gnp* z2zYo7+SVn%lpoW|OARdjvOL^5qS4&n8e;Lus#raN+?1TWidO|Ky9ta{J6geU30?t(5`&XsTZ;3(wPbWDkQK<`Y8@NK&@i55k!w&#q@?Bv81b0D_ubZj7K5 z>JB8dFmX=}NQ4|d=*0vECsJqxF{eFztytpiOL06NEt}I1HRv$1Z2R^8ZLt>Lr?QQQ zBeK9Jq1Qkf-_=n9Vf4dCS&`rR955&A<^)szW1J>J{q)NF6P_OFcl$7+=|EFw3(`r% zH!Op8jj@kqsKzyAa$?>>GBJ%T{F+ANJ!8DW9n9JWj9!!gpEAoeB9>>ZOz0!#U_BE% zHsX_yuDr@nNFsArVhUR&>f_F;=Ka~O-F7AN2LoPlF*NAoB z#=7r!vKAzy8M7{5m-;IApwli5Id$0OB4<;6wedp2M)UOG8Ov*0@U zyh&R-^`k$lNY9 z4y?5a|M5Ed$Bf}oAXqf%$9Bbe2|Uc5s(*Eh9oLRA0U6HiS0S=CP2FKU9-0(bEQ8rh zq$lN@`A0X>Pa$T<#nquTtx(Syg+#I8vvq zKkp;=;0%d5jV5za_bTF-jV;4@R?9f8d~{|= z-4OmHH$tPip+Z&FaV{QL;y12E2!;dvOOHu@&ytQPMJvzzpjYf7GvNRe2cbm?z&vw~ zgN~F3;w<*}+X<3~Gt`Ev^N$xTuH7;`+;3~!H}hpVw^84g*#y7AVi76?u!XUG@H^f{ zK-p8Ee0ug64b5(bjg1q7kpR1{>yN~}m(67@BZ>sU=3^(RJ7YxgV)8HXz(198!a^yE zXWwI*sb5oI?q-Fhi-l1DtQoRq4_Sf!vWLDn286Ma4Bg$THJ6S2IvXf6KY(NjV+CW| z7QuBw*LLn8(pbsss0HJ53r-D5;UQ?gDGVt}M)etznoW?%Z;`VLXujCm8m4?%+y#;v zGeb`vLX{t15mZ*_K}L7s{FUfZl-Rtf$*IbpS3m7y8aPeZk~&WvtQy@&W~IJ*55OyE zt7Nei?A;cRg+T#OrRsW9wU+cZ*@eT7^H#Pq#CitXqWO_r-NL#Ajl0$)i~U)>5zl>& z2&fbATeMt4(>tfwpP(;dt=sIY(7WR5&V6QLrna62^ti=qcMF?xS29*QPTZ$#?d7u5 zR>E&&C???m0sl;xzG4xPDUGlL)rzYc?slF3FWsWo<9OoO<6YNEJs@Wz{tp-H2m9wh zFsHTq_9~8A1;@S4Xo0PuPzsnTaYb4?aly@Zk&?K3w@HOk161$sZZi+T{;~kx1M!0Dj7x|l^NBYxN|WT(QSP$WcqZzr zo@_ysN++5v>cAXnT^;JPY%Si+f5pSuHb`Qtggho0IW-?Ev*_petHq#MedENY^11pb z#pQ&H@rJCUduAI*EMD9`+0^}I@g8i>P zxjHUz@)zl0jSp8Cy0}ep2JfmAYnUK`d0h@s`(84u#@+TFBn$DtynvoIX6)o|DxZ}3=Fx}zbPu}J_r&T0>YRZ+hYrIyQ{up!p zxuLd>4>seXX&jI6-HGUr-CiMC-QthmwXHfJTb_UMpLmNnY7}D5RVfV7l`xzXz8_Yx6pK}d% z?W!2FE6ldzRgn;<+M1D>{j-_Z=onms&#Kw>^1Szo#YVV0gM>z2tJ^JE?(_13m(}Tf z^&2F=DeLwj=xtpa2VO}r=mwm|Dksg0d}|(>4a1AuABLY(fnQ~7usz&8lr=YM+||WUizs*g_R7B7 zu!Fo~VglZckfos`Qkj@%YksdNEcP1L;!j(LEy`3j)V5nOv$@sxr8iZ)cO6)zZLk(L zw^10G`^Bj=NYUiBGXJpqS_T;<=26PRG3@$OaFMZzhk(lkg>Yrx0ySo(E{w+-=h}mM zSy%ezWn+Zw1Z?2MZH27fN#x(7o}6(l_-&zf$9s?La;0tZ^(^WH|W2nE|GmehlW(%uK6{*xLlT z9zbSz^_f3U(WT;H{?y`&@S{$AUY|LZC!QQ*d-^i=@jsKRSk$X#>0-qsOdG}Ui+u|J zqgjhD;dfi?Z<=X1@54ink9_3AaAQNnvAkqPkeqVpg8a)j0g}35+TA1H!5L`*UN`Bg zO2qw;Z421cYult3Mk2<4KY{v-L$5T`ISOZLZR7XG37RXTw-+r{;)@~tjb0J465z&W z@|)l40*|a|$`U_Z-D(xuY}$-&#GDmHXo5;H@lTis3BIr0;@LM;Tu2rFk{b}fc5lqivznJ#N+%GeY-sL-dXiT)`H#T% zv0f0*?dO#e)5)R5atk`~;LK?4suc2Ggy>!;k)6+*hTS52+s4Kwk48hnE)u)!SAlj; z!rxKIR|~*6>)NEF)D6BP|8#4~mG38-$YnGmJI?GGR~)kE4a zYQCoJL8&d?tdGS&o0)L~vz)NeCn{tmb8{6#J`UmyLH?7P$#pJ%evY=m9^-O84iHJ~ z`KDyfa;Tz>vu2yEm^r$dIeHO$*v7(O*jltsT`SP*ZSE8$q5=j=?KrGFoN&qqiK|Bn z60*p^zk36VILbgY1lnNV>`?Qz42uJv8a?G@z})N~Dp^3B$3A+=UY zHYHNYEhK%>>K^&jd6KJ0W7mJ>^YMuailBOH@*cT!X0HX$TBGoDmQnYW=na~xS5zK@ zFXbO&dEMs)1-xCt_&ck1_x<_HUQy;8kD4qP4!Xs&B;AFgTEB0fRJo z-G9SN7`bq1R=?2;y5oFbo&6$aR`N>(5E>R%v0uD9F$u_&jICZIsgYbiuRQx*u&^!t z;qRah{%q$$wX&&Mi1tr`uaT3;=O2r*IqUV2*Q)bjP6W-cu#*!S47>G(QJI9#)5`EJ zSsN7=EO6h&*!;!DpwX!K&)3FkbE%G&LucWLr-?cus@B1&e_rFt2{#6~X7WO00ozQ`PWW5jbm^5KNX{?%`m-Uvl5+LW?M759; zpOGBeQl66eqL?WVw6ql7>F;#;-C5CCpaaqzHe^=QmPsctkKGog|2ElqTQ)pTX?zCN z_%j7nHhp$%vtCbQay>hnuWLu`=+!cB!;%kWSJ{Gy~7TAA@8C?N`cmm;qWNt<5$Cj zOEcA`EZm`7EwcQ?;)2>`7WOB~#L_clFi!m@5ujPJ`(Pi~3 zSrQ$hIcIjbu9Buvs;w|&>RDa2G<_%jFYZ8`=c_t5mLvrkVgWg}!td4SjWdmicN5Wt+HpYJqwlh=02T03|kro`-rSduM@Fp;PbMuz=M<7 zc`p~KDu4GCX|8WAKksRz&s2*)4>0T1>gk+efnl20*jQSyhJVzGTCxwVK4iENvDk`N zdbCim@$);co!Yo~uZ=6YuOdP?%#%_(Zqa)X{?2_A5L5|vNc2XBS=qWp+i~$zJkz}X zy*7A#i?ydo>wNSGNKW2hine{dUMqEwdCp1>j?Orx`VHIyuK9*vibUQZupauqwB_+! z`xF379ZWaw2@;DLDksusr%N8?{s#FrQ6?Fb)|*qTTxDAq==`7Up_S zcgzh`l@xWN{Sqq@gxj^NNz1HW z$VrJ!+Qxgzk_+A1gZv7cqC%^;%S!8l|3c`4;vj$Nwo)*SxI_pwU6xTHbHG%W+3d@V zo*JA!Q-y=Jt3sU8*#JHMtI*=Q^dJ1mL&-sBW);Vj;-Xr4uGC4I#Al1M(j9{nP}hh} zb5|SD*}Y4No)=$6waPhT4x?0tx>>Bv=)-JK4@Xtl*6W>qdJ6N}m$Ec%-yY{P(x#d7 zX-iWLCD+>Q#{!Va6OYQdY292(SHD}31TYb!+3G@Pm@|(M$DH8MQ@)H5Rgv( ztsm~LN_Q!4aw_hcH*X4ObWjCjun$YgPTWvXh^z0~GxYA!=jR5q5T_`F?|w*S9~5kAu!xu$J%Kl$ZuFIjUO+bFa~*ss0#_75sbv{uvoWR?ed z9tN08sIy(ZF}*isTeta?Q!gTO|L*V3M!D!cmrh5|_w2o=6VT3LtL>%q<@mE=yNjIn z%S*G?l#;#w;47NWm6VoRH9wo4ACYq86l~xt>COE(+u3b)>OI%(xLBimD6vb9axc;4 z%FUIhGS;s0dtd1Ab1QxHS7fQ14RQNVQev#Tb~yywvJVEW!G}B5`5Ipg22poilH+6g z_|e1-!Ym62D1+ft_WvcezAJ3BTf+R|v-9J2DO8i!R7rmeHSA;v&o@Dg(VFSd9>vK+b|Of;fUd|&wKwp~@Vf&KFcrn;m(aBp@)1$f zr2es?ERSbXQbk#KY2{{xF-LI8Jm$StnHhh{+5y& z^7({>9wE>o&G}3V@8Ia1JLnmEO`Vy$M@D3Bk{lqGc1bBc-quzOeL+_XSJgur@V4OVL#H7qPX9xnHt@GsDf0uuJuZG1qF3v0fRQ zwUs%0%A@5Wc;@`=f^&bg=`=S{;{L3puYc)ps-KfvN8)D_l>8{?v~~zN6)PzU0yI$@-Mk$?~ygdM3~X%z#CqkK$36mVhDy|V5=pi z9Q+fJWLmwde(f+|+Zhv5?7?;XpEl&OF{TRMQrf(>GW_&H*g5_wKJBD2U5i|NNhLxW z2ltWm@>-3}f**X2iJdw;r{=Fk9>c6#1J3j4U4$+dXZ<^iBx@1Pp!iwOILY{hu_p}Fh+)Z3bXoJ$##sR2PT)vS;zd&DY(!CK zXzMI~i6IJY9b z$V^H}>+2;Fb-9PDb8>DpZf3l&}@}ko-z#{T_HgequG>u>O;sOk>z`|)-|b6~s_ zdAjdlI%yllt>9BbP_~~}k*At;Ku6OdQ$^}tT#+bXA3v;4fm$sgj8DJS_!S(OY1XS7>xFJ@ zf|fJq+Rbi`+NmZ{PCh7LJwqyrwTNCwqvk;(q3LJ|z^8wH|GbPd?x_&ZHDNEz+Es=* zyu|c5l&{+kjq}^nOi(F9oxk*rpYxZ>>+mOSofJ8G_2ciRmF6{tGMh2%pZ$&o>z*ge zq%OQ)^>ydZX(T?wHi9f5u|nd$kgNE6nKgtzBxgqH8-^>RH#HD@KVl&W0AMrnSex9< zv@_xFF}*^2L#^JA1XIzady{y&Dpl;f4Ugq`EH9&>JJ=Dar7UPPhQ>7>kIz#9(t_Gk zW2+@631M6K$gNBD^pUYW^vQ_n2Q;ehw4oYCW8l>!Jn<2 zRcvVIx7-!euS^D=1(UTspQ%bPU%{Pe0sKuWF=()U4dH9VihLAZu|SVHROt~8wBZ*p zM0vh0BXUWuUwaKo<&#(TVGRd;YUM7j_8X*QrqayF3v2X?zDn>3@i3f=`?}nwBS)hL z%|n3Kc6=r$G5;KG^wKU1bwE9<`g8>ls$~YNnts%J%s8@&5N)@peb)W?6OOE9b<(x; zY|letVo4iAH}E?0hgf+~n-Q=1?s=qbVbZ4P(;Cc0&&|`)#OCF9UD0Y8q)`z%Wp=?X zX7`4<7rP_-mOAr$o6vDnmC%pm&_{x-DjNvX^PaY9EH?XDp37=amIR&}n6xh5x593r`#Q)qp1X(A4JYkbjO^6)IreGo{z~@Mg`H|Y7_psOh007e#s37N40G&1t!;pSj ze_!E3!JC>MsbFGNX-0&Xjuo{Jot^kwRiEi5 zF{LmtiYA_`3-e4b$>*_izmX#7F#QYb=hWD5BGzGjWl2rnk@MbQ7HjX!NZ1;YsfK?di>AhHsr1DJB2vg-c_hRr<~6w`A?94<6E}}@2}tb zkoue+>6}>5Yi6N7*$n*h^aK)<5PZAN?&|hIYBZ=Qgw7XYHKZY@Of=qw$9TC4TdqAq zcRV2nf(>|kB*Xqq`zp0;em)TUoQk%0ohv{zTBM)wu&4DIf&>dw(C0(WjUg5Iiav9J zx#1rEWy3A}%>T-hIJNcVsD+cdcSGyu%6h`$2Yg$UMVlGr_!>=ypTRaYNCL2X45= zw#IPC_l-UEdw;oi=`2-)R^$RmGNBhQG)wo5!P7VC@b_RjG=2uFxMq|h@~mKWSDahM z+kHr^n~`3Ev+z)~{Kz!NQiF2p&(eu5(#V|Ilx0h z@WvhHnwqKmfQW%IS#%w9x0D~EN`l?ErFZ~4E3{e6vsW%UEDy{Na6?f^xe z0~U|0>lY^hCJhmF?Q{ONCE-Ey1W_h$yXR~|@dUZh+Au1>IKWF)WXK~N;RUeo!Ej?N zrmG0sivI5}%n~PneJB8K@Jo&P6AT2k!Q-f5fdjq|c&)~WQpQ07(YrYWFm_^v(8k;Zgh{MW9cLSJes2izkAB-!PrbOdl8>9h?JJevm)6E%4y6J1kfk%Ot3FZERL?S!lE5D(sKT}vcz zBi%Q}XDY)e)EhZs&!u>^A!AGcDKJ`KYo=eY+6L8-|U@B8JelP1Hrjv&P!B z**UEp)iam9KAkl0F91C8u`F;$yJF*Eo6UdzI~I78!{5cr2iYI3ws+ne>dwi>i3tbh zP^WcXM}>jZB5pOT%EMvd9sKUJHtOLRSxN8sHJ=7Ykhxg zuMVc^vbZRy@4=M=!WbC zTUahyGI3bo*5di?9lZ`wuF{S14_t4!cqMQ`u7sMdN_kNq;AmlQ)zT>UwOu^%sw;cA zSETU)p{yt346gml)c`{UZcFl84Ky~PW@JAM06j;uSM6h;ctk^HU6$pewr0`zY!|qW zsO_m!6r}6R>`7auNdntzf`SieWQHSTH!?Q9k{{l8R2Xi0R@>D%%xAW3aL-oIuhdV5 z%@PVEn@z-!ig49M2M`c$;!kOmsb<)?TJe1+iBlSCa%{6si-9YW?6mdZ@j5$21%k~F z*YzLn@i|(LHQZw6YrK-|<`vYTEF+6WH;3pN6@zX2o!+8Ya zJ2VJ1z~*xZMFB~1={oW~ur?Rd-V{b8=btvbBi`kSU z&8bj1yBlHjD`e8u*ayk8ZwE}88QCSk9Qb|Lr@*g4NVY`+h_9mLnca@vbU}a}uDX*OuD8yo&81+m?%lRbv~DG7(B{U~G970NeGaYeW2`#TI65 zobcM-6kjCgJ)mD05z=`j#m* zJh*vwe$Lz0IF&KE^1}f=hpA5SAV%=&*i>IncdaU~MW50fC9C8p8i7axLKQxy{N3A6 zYYq0&wv4L&&4T}D%Y{fIlHtQo&VhItg<|VLE5>RPlg?`&=gvkXe_5UQlS77wT+QB~ z)lqtprz^L3zg_O7J4AZbFC8oi-KH@WU==p?4p+g$>oT!RA4NH4!!smz#xB1F1Tbp5 z`-bi@Cs#p`b=rv`Q;y4-Q%niY${i}z{USUzqLZZv(-&tJ!&~wlS>JiWVWAeUFg&LA!weh~+A|~mS*k^XHB)pqu5_Z1wVL$>o@?tLU6oIx zYjHTZ_Lru~we4+<@9MAX>;UAcKIzNT)X<-`4sDHj8hXlqQFEEr=bdDqC4UW5fbVs1 zM(Wm4K4`>R?H^s%Zioujp}Jx{!yDjMwY!s$0@v)Ry!A(u^3<+z0->S=ugn;lG!l%? znp10a?kQN3b5xPmGzcEqKixNecmL+B3qKi|o#Pm#jhlnKYZ5^bDhXUaFQXrZcdfns z649LWI|9B_xQai~NX7NftCgaAGLUJTrzZ8<3DTg|Pw+=IVE~O$)5{NHA7#ym`GRAG z)Ek9NJ5$jKnyZ0gU^e48Jqy3ln( z9kl}fYUv2e9BFI9Y;Gjcd8yf36)Z5RNAl7+&kD?{O^CsHo7sXRhA=%o1@^$`=vx|* ztO^OR9R)EMV0&m!D^=EOF<&z5MYpMQI0{@SQLak`=2*c2dUTBi(yUd{`Zub=#7}n!Jcf< zpD8c3zu$ZdoE4BatPINiq_-E3YkKPXB!k9iJ_!&>q{8VlA3TbF9yyF-ii~6!V??7q zul+`Z^u2+0uai*BrU{k^*r*`e-l#!8o@iq{=A~L2(|tv=uWuO5?J+nw4DF^7A0u7^ zTj2u*Mk^dwjI2*6OmWnDrmy*-oP%pjslb>ehTVSCtyMm1-K Date: Thu, 22 Jun 2017 17:13:09 +0200 Subject: [PATCH 4/7] Add support for graylog stack with filebeat harvesters --- .../filebeat/conf.d/filebeat_apache.conf.yml | 21 ++++++++++++++ .../filebeat/conf.d/filebeat_solr.conf.yml | 16 +++++++++++ .../filebeat/conf.d/filebeat_varnish.conf.yml | 17 +++++++++++ docker-compose-dev-template.yml | 11 +++++++- images/apache_php54/Dockerfile | 3 +- images/apache_php54/bootstrap.sh | 6 +--- images/apache_php56/Dockerfile | 3 +- images/apache_php56/bootstrap.sh | 6 +--- images/apache_php7/Dockerfile | 3 +- images/apache_php7/bootstrap.sh | 6 +--- images/filebeat/Dockerfile | 6 ++++ images/filebeat/service.sh | 28 +++++++++++++++++++ images/solr4/Dockerfile | 5 ++-- images/solr4/bootstrap.sh | 5 +--- images/varnish/Dockerfile | 5 ++-- images/varnish/bootstrap.sh | 5 +--- images/varnish/varnish/default/varnishncsa | 13 +++++++-- stack.sh | 4 +-- 18 files changed, 123 insertions(+), 40 deletions(-) create mode 100644 config/graylog/filebeat/conf.d/filebeat_apache.conf.yml create mode 100644 config/graylog/filebeat/conf.d/filebeat_solr.conf.yml create mode 100644 config/graylog/filebeat/conf.d/filebeat_varnish.conf.yml create mode 100644 images/filebeat/Dockerfile create mode 100644 images/filebeat/service.sh diff --git a/config/graylog/filebeat/conf.d/filebeat_apache.conf.yml b/config/graylog/filebeat/conf.d/filebeat_apache.conf.yml new file mode 100644 index 0000000..297fd31 --- /dev/null +++ b/config/graylog/filebeat/conf.d/filebeat_apache.conf.yml @@ -0,0 +1,21 @@ +###################### Filebeat Configuration for Apache2 ######################### + +# Retrieve logs from services +# @author Andréas 'Screamz' HANSS + +#=========================== Filebeat prospectors ============================= + +filebeat.prospectors: +- input_type: log + paths: + - /app-logs/apache/access.log + document_type: apache_access + +- input_type: log + paths: + - /app-logs/apache/error.log + document_type: apache_error + +#----------------------------- Graylog2 output -------------------------------- +output.logstash: + hosts: ["graylog-server:5044"] diff --git a/config/graylog/filebeat/conf.d/filebeat_solr.conf.yml b/config/graylog/filebeat/conf.d/filebeat_solr.conf.yml new file mode 100644 index 0000000..8910b1e --- /dev/null +++ b/config/graylog/filebeat/conf.d/filebeat_solr.conf.yml @@ -0,0 +1,16 @@ +###################### Filebeat Configuration for Apache SOLR ######################### + +# Retrieve logs from services +# @author Andréas 'Screamz' HANSS + +#=========================== Filebeat prospectors ============================= + +filebeat.prospectors: +- input_type: log + paths: + - /app-logs/solr/solr.log + document_type: solr + +#----------------------------- Graylog2 output -------------------------------- +output.logstash: + hosts: ["graylog-server:5047"] diff --git a/config/graylog/filebeat/conf.d/filebeat_varnish.conf.yml b/config/graylog/filebeat/conf.d/filebeat_varnish.conf.yml new file mode 100644 index 0000000..0f4c55d --- /dev/null +++ b/config/graylog/filebeat/conf.d/filebeat_varnish.conf.yml @@ -0,0 +1,17 @@ +###################### Filebeat Configuration for Varnish NCSA ######################### + +# Retrieve logs from services +# @author Andréas 'Screamz' HANSS + +#=========================== Filebeat prospectors ============================= + +filebeat.prospectors: +- input_type: log + paths: + - /app-logs/varnish/varnishncsa.log + document_type: varnish + include_lines: ['BAN', 'PURGE'] + +#----------------------------- Graylog2 output -------------------------------- +output.logstash: + hosts: ["graylog-server:5046"] diff --git a/docker-compose-dev-template.yml b/docker-compose-dev-template.yml index 91728f5..274418a 100644 --- a/docker-compose-dev-template.yml +++ b/docker-compose-dev-template.yml @@ -175,4 +175,13 @@ services: - mongo - elasticsearch ports: - - "9000:9000" \ No newline at end of file + - "9000:9000" + filebeats: + container_name: graylog-filebeat + #image: TODO_ON_DOCKERHUB_UPLOAD + build: ./images/filebeat + depends_on: + - graylog + volumes: + - ./config/graylog/filebeat:/mnt + - ./logs:/app-logs diff --git a/images/apache_php54/Dockerfile b/images/apache_php54/Dockerfile index 3c4e210..b1e608c 100644 --- a/images/apache_php54/Dockerfile +++ b/images/apache_php54/Dockerfile @@ -24,8 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless \ - netcat; + openjdk-7-jre-headless; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php54/bootstrap.sh b/images/apache_php54/bootstrap.sh index 69b0493..b31394a 100644 --- a/images/apache_php54/bootstrap.sh +++ b/images/apache_php54/bootstrap.sh @@ -55,12 +55,8 @@ trap clean_up SIGTERM service apache2 restart -# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) -tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & -tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & - echo [`date`] Bootstrap finished tail -f /dev/null & child=$! -wait "$child" \ No newline at end of file +wait "$child" diff --git a/images/apache_php56/Dockerfile b/images/apache_php56/Dockerfile index 627b5ce..2c761ad 100644 --- a/images/apache_php56/Dockerfile +++ b/images/apache_php56/Dockerfile @@ -24,8 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless \ - netcat; + openjdk-7-jre-headless; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php56/bootstrap.sh b/images/apache_php56/bootstrap.sh index 598176c..dd5336c 100644 --- a/images/apache_php56/bootstrap.sh +++ b/images/apache_php56/bootstrap.sh @@ -56,12 +56,8 @@ trap clean_up SIGTERM service apache2 restart -# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) -tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & -tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & - echo [`date`] Bootstrap finished tail -f /dev/null & child=$! -wait "$child" \ No newline at end of file +wait "$child" diff --git a/images/apache_php7/Dockerfile b/images/apache_php7/Dockerfile index 5069150..bea14ab 100644 --- a/images/apache_php7/Dockerfile +++ b/images/apache_php7/Dockerfile @@ -25,8 +25,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ imagemagick \ jpegoptim \ poppler-utils \ - openjdk-7-jre-headless \ - netcat; + openjdk-7-jre-headless; # locale for date, time & currency php functions # q: for what is the locale setup needed ? diff --git a/images/apache_php7/bootstrap.sh b/images/apache_php7/bootstrap.sh index 598176c..dd5336c 100644 --- a/images/apache_php7/bootstrap.sh +++ b/images/apache_php7/bootstrap.sh @@ -56,12 +56,8 @@ trap clean_up SIGTERM service apache2 restart -# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) -tail -f /var/log/apache2/access.log |nc -w 1 -u graylog-server 5557 & -tail -f /var/log/apache2/error.log |nc -w 1 -u graylog-server 5557 & - echo [`date`] Bootstrap finished tail -f /dev/null & child=$! -wait "$child" \ No newline at end of file +wait "$child" diff --git a/images/filebeat/Dockerfile b/images/filebeat/Dockerfile new file mode 100644 index 0000000..5f6db7a --- /dev/null +++ b/images/filebeat/Dockerfile @@ -0,0 +1,6 @@ +FROM docker.elastic.co/beats/filebeat:5.4.2 +USER root +COPY service.sh /root/service.sh +RUN chmod +x /root/service.sh && mkdir /usr/share/filebeat/conf.d/ +WORKDIR /usr/share/filebeat/ +CMD ["/root/service.sh"] diff --git a/images/filebeat/service.sh b/images/filebeat/service.sh new file mode 100644 index 0000000..a4a5a50 --- /dev/null +++ b/images/filebeat/service.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# Copy config to have file with root owner +cp /mnt/conf.d/* /usr/share/filebeat/conf.d/ +cd /usr/share/filebeat/conf.d/ + +configFileAmount=$(ls -l *.conf.yml | egrep -c '^-') +currentFile=0 + +# Avoid troll config +if [ $configFileAmount -gt 25 ];then + echo "Too much config files detected" + exit 1 +fi + +# Create a worker for each defined service, each one need his own registery so we make a folder for each +for f in *.conf.yml;do + (( currentFile++ )) + if [ $currentFile -eq $configFileAmount ];then + mkdir "workdir-$f" && mv $f "workdir-$f" && cd "workdir-$f" + exec filebeat -c $f + cd .. + else + mkdir "workdir-$f" && mv $f "workdir-$f" && cd "workdir-$f" + filebeat -c $f & + cd .. + fi +done diff --git a/images/solr4/Dockerfile b/images/solr4/Dockerfile index 59aa008..780ae52 100644 --- a/images/solr4/Dockerfile +++ b/images/solr4/Dockerfile @@ -13,8 +13,7 @@ RUN echo $DOCKER_TIMEZONE > /etc/timezone; dpkg-reconfigure -f noninteractive tz # Base packages # ----------------------------------------------------------------------------- RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -qqy \ - openjdk-7-jre-headless \ - netcat; + openjdk-7-jre-headless; COPY ./solr_4.10 /opt/solr COPY init/solr /etc/init.d/solr @@ -32,4 +31,4 @@ EXPOSE 8983 WORKDIR /opt/solr -CMD ["/root/bootstrap.sh"] \ No newline at end of file +CMD ["/root/bootstrap.sh"] diff --git a/images/solr4/bootstrap.sh b/images/solr4/bootstrap.sh index cfe36fe..97d65ed 100755 --- a/images/solr4/bootstrap.sh +++ b/images/solr4/bootstrap.sh @@ -15,11 +15,8 @@ trap clean_up SIGTERM service solr start -# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) -tail -f /opt/solr/logs/solr.log | nc -w 1 -u graylog-server 5556 & - echo [`date`] Bootstrap finished tail -f /dev/null & child=$! -wait "$child" \ No newline at end of file +wait "$child" diff --git a/images/varnish/Dockerfile b/images/varnish/Dockerfile index 94bba76..c483e83 100644 --- a/images/varnish/Dockerfile +++ b/images/varnish/Dockerfile @@ -19,8 +19,7 @@ RUN apt-get update && \ vim \ htop \ procps \ - net-tools \ - netcat; + net-tools; # varnish # ----------------------------------------------------------------------------- @@ -46,4 +45,4 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* EXPOSE 81 EXPOSE 6082 -CMD ["/root/bootstrap.sh"] \ No newline at end of file +CMD ["/root/bootstrap.sh"] diff --git a/images/varnish/bootstrap.sh b/images/varnish/bootstrap.sh index f341b8b..68690e4 100755 --- a/images/varnish/bootstrap.sh +++ b/images/varnish/bootstrap.sh @@ -21,11 +21,8 @@ service varnishncsa start varnish-agent -# Hook to dispatch logs in graylog (Provisional, awaiting smarter & cleaner config) -/usr/bin/varnishncsa -a -P /var/run/varnishncsa/varnishncsa.pid -F "{\"method\": \"%m\", \"X-Match\": \"%{X-Match}i\", \"X-Location-Id\": \"%{X-Location-Id}i\"}" | grep -P --line-buffered "(BAN|PURGE)" | nc -w 1 -u graylog-server 5555 & - echo [`date`] Bootstrap finished tail -f /dev/null & child=$! -wait "$child" \ No newline at end of file +wait "$child" diff --git a/images/varnish/varnish/default/varnishncsa b/images/varnish/varnish/default/varnishncsa index c33ec13..4ea9c8a 100644 --- a/images/varnish/varnish/default/varnishncsa +++ b/images/varnish/varnish/default/varnishncsa @@ -11,7 +11,7 @@ # NCSA log format, to be used by HTTP log analyzers VARNISHNCSA_ENABLED=1 -LOG_FORMAT="%h|%u|%{%Y-%m-%d}t|%{%H:%M:%S}t|%{%z}t|%m|%{Host}i|%U|%q|%s|%b|%{Referer}i|%{User-agent}i|%{Varnish:time_firstbyte}x|%T|%D|%{Varnish:handling}x|%{X-FE-Varnish-Obj-TTL}o|%{X-FE-Varnish-Backend}o|%{X-FE-Varnish-Obj-Stat}o|%{X-Location-Id}i|%{X-PropReference-Id}i" +LOG_FORMAT="%h~%u~%{%Y-%m-%d}t~%{%H:%M:%S}t~%{%z}t~%m~%{Host}i~%U~%q~%s~%b~%{Referer}i~%{User-agent}i~%{Varnish:time_firstbyte}x~%T~%D~%{Varnish:handling}x~%{X-FE-Varnish-Obj-TTL}o~%{X-FE-Varnish-Backend}o~%{X-FE-Varnish-Obj-Stat}o~%{X-Location-Id}i~%{X-Match}i" DAEMON_OPTS="-a -w /var/log/varnish/varnishncsa.log -D -P /var/run/varnishncsa/varnishncsa.pid -F ${LOG_FORMAT}" @@ -83,4 +83,13 @@ DAEMON_OPTS="-a -w /var/log/varnish/varnishncsa.log -D -P /var/run/varnishncsa/v # # 20 %{X-FE-Varnish-Obj-Stat}o # Custom header showing whether request resulted in cache HIT -# or MISS, if HIT, the number of HITs thus far for the object. \ No newline at end of file +# or MISS, if HIT, the number of HITs thus far for the object. +# +# 21 %{X-Location-Id}i +# +# Custom header showing locationID of banned eZPublish contents. +# +# 22 %{X-Match}i +# +# Custom header displaying a url path that will be banned. +# diff --git a/stack.sh b/stack.sh index 3709627..8dffb4a 100755 --- a/stack.sh +++ b/stack.sh @@ -211,7 +211,7 @@ case "$1" in start|run) $DOCKER_COMPOSE -p "$DOCKER_PROJECT_NAME" down - $DOCKER_COMPOSE pull + #$DOCKER_COMPOSE pull $DOCKER_COMPOSE -p "$DOCKER_PROJECT_NAME" up -d ;; @@ -255,4 +255,4 @@ case "$1" in # any other variation, let it go directly through to docker-compose $DOCKER_COMPOSE -p "$DOCKER_PROJECT_NAME" $@ -esac \ No newline at end of file +esac From 677ddab09ff12a5d310acf59422c4d6d3a77ca7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9as=20Hanss?= Date: Thu, 22 Jun 2017 17:18:37 +0200 Subject: [PATCH 5/7] Put again pull on start --- stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.sh b/stack.sh index 2a5d5ab..1be03ae 100755 --- a/stack.sh +++ b/stack.sh @@ -219,7 +219,7 @@ case "$1" in start|run) $DOCKER_COMPOSE -p "$DOCKER_PROJECT_NAME" down - #$DOCKER_COMPOSE pull + $DOCKER_COMPOSE pull $DOCKER_COMPOSE -p "$DOCKER_PROJECT_NAME" up -d ;; From 5e19a8f65811b93a7671a0520f766af99091059f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9as=20Hanss?= Date: Fri, 23 Jun 2017 10:16:37 +0200 Subject: [PATCH 6/7] Make graylog as optionnal dependency --- docker-compose-dev-graylog-template.yml | 173 ++++++++++++++++++++++++ docker-compose-dev-template.yml | 32 ----- 2 files changed, 173 insertions(+), 32 deletions(-) create mode 100644 docker-compose-dev-graylog-template.yml diff --git a/docker-compose-dev-graylog-template.yml b/docker-compose-dev-graylog-template.yml new file mode 100644 index 0000000..725928d --- /dev/null +++ b/docker-compose-dev-graylog-template.yml @@ -0,0 +1,173 @@ + +version: '2' +services: + web: + image: klabs/apache_${DOCKER_PHP_VERSION} + hostname: ${DOCKER_PROJECT_NAME}_web + container_name: web + ports: + - "82:82" + - "88:88" + volumes: + - ./config/apache/sites-available/001-dynamic-vhost-ez5.conf:/etc/apache2/sites-available/001-dynamic-vhost-ez5.conf + - ./config/apache/sites-available/002-dynamic-vhost-ezplatform.conf:/etc/apache2/sites-available/002-dynamic-vhost-ezplatform.conf + - ./config/apache/sites-available/003-dynamic-vhost-ez4.conf:/etc/apache2/sites-available/003-dynamic-vhost-ez4.conf + - ./config/apache/sites-available/ez5-common.conf:/etc/apache2/sites-available/ez5-common.conf + - ./config/apache/sites-available/ezplatform-common.conf:/etc/apache2/sites-available/ezplatform-common.conf + - ./logs/apache/:/var/log/apache2 + - "./config/apache/php5/custom_vars.ini:$DOCKER_PHP_CONF_PATH/apache2/conf.d/custom_vars.ini" + - "./config/apache/php5/timezone.ini:$DOCKER_PHP_CONF_PATH/apache2/conf.d/timezone.ini" + - "$DOCKER_WWW_ROOT:$DOCKER_WWW_DEST" + - "$DOCKER_STORAGE_LOCAL_PATH:$DOCKER_STORAGE_MOUNT_POINT" + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + cli: + image: klabs/cli_${DOCKER_PHP_VERSION} + hostname: ${DOCKER_PROJECT_NAME}_cli + container_name: cli + privileged: true + volumes: + - ./logs/cli:/var/log/php + # files in /tmp/cron.d get managed by bootstrap.sh + - ./config/cli/cron.d:/tmp/cron.d + - ~/.gitconfig:/home/site/.gitconfig + - ~/.ssh/config:/home/site/.ssh/config + - "$SSH_AUTH_SOCK:/ssh-agent" + - "./config/cli/php5/custom_vars.ini:$DOCKER_PHP_CONF_PATH/cli/conf.d/custom_vars.ini" + - "./config/cli/php5/timezone.ini:$DOCKER_PHP_CONF_PATH/cli/conf.d/timezone.ini" + - "$DOCKER_WWW_ROOT:$DOCKER_WWW_DEST" + - "$DOCKER_STORAGE_LOCAL_PATH:$DOCKER_STORAGE_MOUNT_POINT" + environment: + - SSH_AUTH_SOCK=/ssh-agent + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + memcached: + image: klabs/memcached + hostname: ${DOCKER_PROJECT_NAME}_memcached + container_name: memcached + ports: + - "11211:11211" + volumes: + - ./logs/memcache/:/var/log/memcache + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + solr: + image: klabs/solr4 + hostname: ${DOCKER_PROJECT_NAME}_solr + container_name: solr + ports: + - "8983:8983" + volumes: + - "${DOCKER_SOLR_CONF_PATH}:/opt/solr/solr/" + - ./data/solr/:/opt/solr/data/ + - ./logs/solr/:/opt/solr/logs + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + varnish: + image: klabs/varnish + hostname: ${DOCKER_PROJECT_NAME}_varnish + container_name: varnish + ports: + - "81:81" + volumes: + - "${DOCKER_VARNISH_VCL_FILE}:/etc/varnish/default.vcl" + - ./logs/varnish/:/var/log/varnish + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + # varnish init daemon wants to set ulimits + - SYS_RESOURCE + security_opt: + - apparmor:unconfined + depends_on: + - web + - cli + haproxy: + image: klabs/haproxy + hostname: ${DOCKER_PROJECT_NAME}_haproxy + container_name: haproxy + ports: + - "80:80" + volumes: + - ./config/haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg + env_file: + - docker-compose.env + - docker-compose.env.local + depends_on: + - web + - varnish + - solr + mysql: + image: klabs/mysql + hostname: ${DOCKER_PROJECT_NAME}_mysql + container_name: mysql + # nb: this prevents the image to start on some ubuntu installs because of apparmor config... + #privileged: true + ports: + - "3307:3306" + volumes: + - ./config/mysql/:/etc/mysql/conf.d/ + - ./data/mysql/:/var/lib/mysql + - ./logs/mysql/:/var/log/mysql + env_file: + - docker-compose.env + - docker-compose.env.local + cap_add: + - SYS_PTRACE + command: ["/root/bootstrap.sh", "mysqld --character-set-server=utf8 --collation-server=utf8_unicode_ci"] + + # official phpmyadmin image + phpmyadmin: + image: phpmyadmin/phpmyadmin + hostname: ${DOCKER_PROJECT_NAME}_phpmyadmin + container_name: phpmyadmin + env_file: + - docker-compose.env + - docker-compose.env.local + depends_on: + - mysql + # Graylog stack + mongo: + container_name: graylog-mongo + image: "mongo:3" + volumes: + - ./config/graylog:/docker-entrypoint-initdb.d + elasticsearch: + container_name: graylog-elastic + image: "elasticsearch:2" + command: "elasticsearch -Des.cluster.name='graylog'" + graylog: + container_name: graylog-server + image: 'graylog2/server:latest' + environment: + GRAYLOG_PASSWORD_SECRET: somepasswordpepper + GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 + GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api + depends_on: + - mongo + - elasticsearch + ports: + - "9000:9000" + filebeats: + container_name: graylog-filebeat + #image: TODO_ON_DOCKERHUB_UPLOAD + build: ./images/filebeat + depends_on: + - graylog + volumes: + - ./config/graylog/filebeat:/mnt + - ./logs:/app-logs diff --git a/docker-compose-dev-template.yml b/docker-compose-dev-template.yml index 274418a..87fa7b0 100644 --- a/docker-compose-dev-template.yml +++ b/docker-compose-dev-template.yml @@ -153,35 +153,3 @@ services: - docker-compose.env.local depends_on: - mysql - - # Graylog stack - mongo: - container_name: graylog-mongo - image: "mongo:3" - volumes: - - ./config/graylog:/docker-entrypoint-initdb.d - elasticsearch: - container_name: graylog-elastic - image: "elasticsearch:2" - command: "elasticsearch -Des.cluster.name='graylog'" - graylog: - container_name: graylog-server - image: 'graylog2/server:latest' - environment: - GRAYLOG_PASSWORD_SECRET: somepasswordpepper - GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 - GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api - depends_on: - - mongo - - elasticsearch - ports: - - "9000:9000" - filebeats: - container_name: graylog-filebeat - #image: TODO_ON_DOCKERHUB_UPLOAD - build: ./images/filebeat - depends_on: - - graylog - volumes: - - ./config/graylog/filebeat:/mnt - - ./logs:/app-logs From 64d28df48baa4271ca0d7ae912002eb9b8c542f3 Mon Sep 17 00:00:00 2001 From: Steven Bressey Date: Tue, 27 Jun 2017 09:00:35 +0200 Subject: [PATCH 7/7] added graylog include file --- graylog.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 graylog.yml diff --git a/graylog.yml b/graylog.yml new file mode 100644 index 0000000..ce641f8 --- /dev/null +++ b/graylog.yml @@ -0,0 +1,36 @@ +# Graylog stack +version: '2' +services: + mongo: + container_name: graylog-mongo + image: "mongo:3" + volumes: + - ./config/graylog:/docker-entrypoint-initdb.d + + elasticsearch: + container_name: graylog-elastic + image: "elasticsearch:2" + command: "elasticsearch -Des.cluster.name='graylog'" + + graylog: + container_name: graylog-server + image: 'graylog2/server:latest' + environment: + GRAYLOG_PASSWORD_SECRET: somepasswordpepper + GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 + GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api + depends_on: + - mongo + - elasticsearch + ports: + - "9000:9000" + + filebeats: + container_name: graylog-filebeat + #image: TODO_ON_DOCKERHUB_UPLOAD + build: ./images/filebeat + depends_on: + - graylog + volumes: + - ./config/graylog/filebeat:/mnt + - ./logs:/app-logs \ No newline at end of file