From ad9662899c6cae86e6b0f04e9264816b0833dd91 Mon Sep 17 00:00:00 2001 From: Haythem REHOUMA Date: Wed, 8 May 2024 23:22:20 -0400 Subject: [PATCH] Add files via upload --- .../2- ATLIER #1 KAFKA-WINDOWS.txt" | 47 ++++++++++++++++++ .../3- ATLIER #1 KAFKA-LINUX2.txt" | 47 ++++++++++++++++++ .../KAFKA DEMO - Commandes VSCODE.docx" | Bin 0 -> 7030 bytes .../pr\303\251sentationKAFKA/consumer.py" | 23 +++++++++ .../pr\303\251sentationKAFKA/producer.py" | 30 +++++++++++ 5 files changed, 147 insertions(+) create mode 100644 "A19 - Kafka/pr\303\251sentationKAFKA/2- ATLIER #1 KAFKA-WINDOWS.txt" create mode 100644 "A19 - Kafka/pr\303\251sentationKAFKA/3- ATLIER #1 KAFKA-LINUX2.txt" create mode 100644 "A19 - Kafka/pr\303\251sentationKAFKA/KAFKA DEMO - Commandes VSCODE.docx" create mode 100644 "A19 - Kafka/pr\303\251sentationKAFKA/consumer.py" create mode 100644 "A19 - Kafka/pr\303\251sentationKAFKA/producer.py" diff --git "a/A19 - Kafka/pr\303\251sentationKAFKA/2- ATLIER #1 KAFKA-WINDOWS.txt" "b/A19 - Kafka/pr\303\251sentationKAFKA/2- ATLIER #1 KAFKA-WINDOWS.txt" new file mode 100644 index 0000000..19d37c2 --- /dev/null +++ "b/A19 - Kafka/pr\303\251sentationKAFKA/2- ATLIER #1 KAFKA-WINDOWS.txt" @@ -0,0 +1,47 @@ +*********************************** +**************ATLIER KAFKA********* +*********************************** +*****1- Démarrez ZOOKEEPER +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\zookeeper-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\zookeeper.properties + +*****2- Démarrez le serveur KAFKA +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\kafka-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\server.properties + +*****3- Création d'un premier topic (ouvrir CMD dans windows) +kafka-topics --bootstrap-server localhost:9092 --topic justine --create --partitions 1 --replication-factor 1 + +*****4- Création de plusieurs topics (ouvrir CMD dans windows) +kafka-topics --bootstrap-server localhost:9092 --topic myfirst --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mysecond --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mythird --create --partitions 1 --replication-factor 1 + +*****5- lister les topics +kafka-topics --bootstrap-server localhost:9092 -list + +*****6- décrire un topic spécéfique +kafka-topics --bootstrap-server localhost:9092 --describe --topic justine +kafka-topics --bootstrap-server localhost:9092 --describe --topic myfirst +kafka-topics --bootstrap-server localhost:9092 --describe --topic mysecond +kafka-topics --bootstrap-server localhost:9092 --describe --topic mythird + +*****7- Création d'un 4ème topic avec un facteur de répartition =3 (ouvrir CMD dans windows) +kafka-topics --bootstrap-server localhost:9092 --topic myfourth --create --partitions 3 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --describe --topic myfourth + + +*****8- Création d'un 5ème topic avec un facteur de réplication =3 (ouvrir CMD dans windows) +kafka-topics --bootstrap-server localhost:9092 --topic myfifth --create --partitions 1 --replication-factor 3 + + +***Toutes les commandes +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\zookeeper-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\zookeeper.properties +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\kafka-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\server.properties +kafka-topics --bootstrap-server localhost:9092 --topic justine --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic myfirst --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mysecond --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mythird --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 -list +kafka-topics --bootstrap-server localhost:9092 --describe --topic justine +kafka-topics --bootstrap-server localhost:9092 --describe --topic myfirst +kafka-topics --bootstrap-server localhost:9092 --describe --topic mysecond +kafka-topics --bootstrap-server localhost:9092 --describe --topic mythird \ No newline at end of file diff --git "a/A19 - Kafka/pr\303\251sentationKAFKA/3- ATLIER #1 KAFKA-LINUX2.txt" "b/A19 - Kafka/pr\303\251sentationKAFKA/3- ATLIER #1 KAFKA-LINUX2.txt" new file mode 100644 index 0000000..73fe0c1 --- /dev/null +++ "b/A19 - Kafka/pr\303\251sentationKAFKA/3- ATLIER #1 KAFKA-LINUX2.txt" @@ -0,0 +1,47 @@ +*********************************** +**************ATLIER KAFKA********* +*********************************** +*****1- Démarrez ZOOKEEPER +bin/zookeeper-server-start.sh config/zookeeper.properties + +*****2- Démarrez le serveur KAFKA +bin/kafka-server-start.sh config/server.properties + +*****3- Création d'un premier topic (ouvrir CMD dans windows) +./kafka-topics.sh --bootstrap-server localhost:9092 --topic justine --create --partitions 1 --replication-factor 1 + +*****4- Création de plusieurs topics (ouvrir CMD dans windows) +./kafka-topics.sh --bootstrap-server localhost:9092 --topic myfirst --create --partitions 1 --replication-factor 1 +./kafka-topics.sh --bootstrap-server localhost:9092 --topic mysecond --create --partitions 1 --replication-factor 1 +./kafka-topics.sh --bootstrap-server localhost:9092 --topic mythird --create --partitions 1 --replication-factor 1 + +*****5- lister les topics +./kafka-topics.sh --bootstrap-server localhost:9092 -list + +*****6- décrire un topic spécéfique +./kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic justine +./kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic myfirst +./kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic mysecond +./kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic mythird + +*****7- Création d'un 4ème topic avec un facteur de répartition =3 (ouvrir CMD dans windows) +./kafka-topics.sh --bootstrap-server localhost:9092 --topic myfourth --create --partitions 3 --replication-factor 1 +./kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic myfourth + + +*****8- Création d'un 5ème topic avec un facteur de réplication =3 (ouvrir CMD dans windows) +kafka-topics --bootstrap-server localhost:9092 --topic myfifth --create --partitions 1 --replication-factor 3 + + +***Toutes les commandes +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\zookeeper-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\zookeeper.properties +C:\HADOOP\kafka_2.12-3.5.0\bin\windows\kafka-server-start.bat C:\HADOOP\kafka_2.12-3.5.0\config\server.properties +kafka-topics --bootstrap-server localhost:9092 --topic justine --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic myfirst --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mysecond --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 --topic mythird --create --partitions 1 --replication-factor 1 +kafka-topics --bootstrap-server localhost:9092 -list +kafka-topics --bootstrap-server localhost:9092 --describe --topic justine +kafka-topics --bootstrap-server localhost:9092 --describe --topic myfirst +kafka-topics --bootstrap-server localhost:9092 --describe --topic mysecond +kafka-topics --bootstrap-server localhost:9092 --describe --topic mythird diff --git "a/A19 - Kafka/pr\303\251sentationKAFKA/KAFKA DEMO - Commandes VSCODE.docx" "b/A19 - Kafka/pr\303\251sentationKAFKA/KAFKA DEMO - Commandes VSCODE.docx" new file mode 100644 index 0000000000000000000000000000000000000000..80889ff7c724671d138cef42ab9cb2e67a59bb70 GIT binary patch literal 7030 zcma)h1ymf{vi3j*ConjH;O_1Y!5Ih^+}+(naM$3$-Q9z`1PvA(f&^!f z@$P|;C~*CQmNUo^wm#~&-CiK3Z!F2UcG31X@bLu?6+ocs8G>pAz+(kBh6`f*ohXWQ z3lOVO%qC=u&7E>%s~LbPte=C`-I|Wp%TNb)17E7X?3AegI6x<6lgY5)~#i_J;vi?m7<8Gvf?tWBC!Umnc+DF6?;c=1bxZDFMM-&lgzLUcb z`>7}fLYM)g_dE}zx&#s zKYuL?2QQ5w=3@>800_VU0I&ZsPiX(nlaqK0@}^I&XeZE zb@5cMIjYW_5&)15H-fxH;x@OBs0Nc}^M${f7D@TU2~8n`g#_=g-a6zCuu{k-R4CPF zqto>m>*NYm2_>yf_VC66o8#(VnOT0GJ{~brQ{PX&szp+WZ3H#HigId^LXW_w?W#;D zoWiFw0do6rxR7?e)N1dOF}FYL;&$30SD0qNV<~-A@xDJ)9Wgknw$#yEDyKBufcHKS zo{b6uIxDxQD|Swf49i*mlAUjp(&$HdT;O4ZL0l9=xIylIa&VDu-6y<*&u|@EtQ>5f z$J{ViHWA=JYPuH*dl^b&D39=kM~nhGC7t=se%ZHhnJ&xAdJzVTi2SnWj3)@WaP3gP zD}7B3%I)IZrp>gj@H%)sPur@{uMN%*pP3PGEdZ=Mc^?NMR$zY4R$>-){ydDi_<1OI z{-yGRb<%3&NCy;{tm0m1Q6W#0V9?te^=J$~#YPZr=$ZbG8gUL*{c?Mc0VF}!xgX&t zrq7si!o@d(k^bp-^C{g-OT<12#6Q!V8bErOIqfAJ*6$p=KW))caoq}rcKFVTEsA9% zhSs$>XiT`>OZ*h=&_;0)Ts(_lAhJ$I(m%XPzam6c90|A_fRLD@Dr&*m5Hx@%ITe>+ zZI-OYG?aTk@c-ZpG^$bycbZi^qgz~)HWKPlv%Erms-ft*-h3En0AL^Xztzxxux@H+ z>#Sm6Xl?SVh!TFsdOkcTJXHlAi0|431v@BRzDQ_w$bNCz%r5l$3F`^Ju#Qv3MWiXt zCnBFaJa74-S_(Hhh+TDHRhI~-iQbV=4hk`9XkM4QahPe&Ml<)65z%o|X^icSm)$hC zYa2G+GI|AbjykEu$=AeE^=CHtt?R`M?l+8y7q{1}(zfa$-H{x|Qz$i)aTvVxcOr3y z8ZwM6l2zF>9g@S(&LogFJ6{^1(_Z)v%ucmr7Vb=m)*O?xc5>ECm+F04q_Kq&!5XRvGz?Go=75vWg!>M`6`9M$1*-COhu@$_~x}$pXZXuRuw}fw%dB8ng;fWi{OGyTi z=&?JIWTkvNa^%_0%Jhf8KGtnB5?O9){$5~N)DiFaJgu?1eu2L{Qdk`@#9x|IZNTKy zH>a1m8;YKxj1Er8eAtBuxwH}HNUxkxB0u3eX_QiU&pMlFhnKIW_R*4TH*-MH8PKce zBHqXHPo{>2B4Lx#G<;`-lw*d-!}&_&p6$lmd@|+xwIMy!o_g&Xj?0I-)Qtls?UGs7g7O8-j+ z-b5!bO;0r_);dC^!^$K?zCt_GXx+T5kz4*KO`m2hsdY`##taTc5`tkb>nD2$9jWdf z@+}sf0<6p*ma9=;J1CGRgEFTaOETc+!oa+Ha$rI2h6kVDl7#Cjcgm4C>%Js4EVS4L z-L9zJRKNVGB?_RE(D;#=vsG%KvS{d_LM~n`lnudutkAVHx+3?Kj$#o|Vi-ivWb77n z2%CJEK8bGG+rS5Bupcof!Ex{d4Q4K{WYBt5@wg-=H5g1Z*hq?AqKIA!a&8R^8Z|L(#-dS{wtOFLIz7&) zZ}Xy}@`VruLVPH%kMrq%3e0aFPWv7;bzLd|yqrOoyUN{{3JACB6&q6Xn*r5>4^-CL zNVf{>o%#iKqEUSErgxJwgI+UhK5#D@vR@Jeo$+Lpyh&9Z41wMM*dCgBQiW7mFmE8{ z&cG!N=>pgOq+oi0egfL(0VYBEX8^#&<0klj=(5IkMlLocw$4u*EV8PDT~G&F$C)u>O&y&D@jvVOs%kn`6P+B)8Hb#ncUVT38vOet~ez60#**6o0U?V4!Q%w8T@nmTy z+X`*k?LC!m!un~Y)g`+Y{GttevjrK>8E?)|d=N&DHnnja7@yI&NwDHWU%~5j7T4+& zj!4tGJPJc8h4T+zz^`Vstyz19Uj?eR7_)X$$`D#SZluNVx@@wG~@fR#;JFHDaIO7!iV;lfv@lmVt z>7Z@S>C9%3mlF#K4Rh15#%9mzO)!E@XQ5(;mQ&q5{^rJ2PBy3qRMzN<@^#;7u(u4eYG!N$}b16N@J*Xkg;Vm zNpaU^M+nf-i(sJ5A*`k95A=JeRK4gtulmqq@aa3KhMNm2Vy!61E2G(Wjp%f{4Gv(d zsdm8LshKCjb!JEUbIYAZ>}3sKKEYd|4!lfVU0esl$qg_iwhhL$vwD2f@h>j4pssIT698A8Y^#M zomeaZ>0%SFuGTYSwF0RPX%~mzZGS2FnB|zkdIsZ3p#UdlKSQuGgh_OB22socW{PF^ zO9q%bKGU#V=8sWgTjE(F61NFz7hwClvj5X5pNfepA;p z2`?o0QzsT1s3iq~#<_yQ(AHMz_$sm}r9%Vz^rg&UT;0<_68i~gHy~UG3aYfK+0@i; zgh!FRUo^Sj_HLFz2D#MM4^OrZL2Z1zoiB8WsD`g-(HpBWlVrM&H<#PPKk2=m)tjHw z=Uf_dU!vrSDTD{B9BZ%~Yw#cw7k2CuZb`q-Dw}nDXnS*E+7MRlEreHZRYG(`OnfwR z2JY)>wf92k+j$N-&-|KbnA72bos=BdDOZ&u;-ovfXI(UGtfH*Bpps;C9fsE-!iW^* zY{=uFLmm4tWtayK=ZGw<81Ss1dh~fvYOxb{3Vl+J&GqqgzY^L!7v_G=H2(J=gNS`! z0-E*wu026qn87tm#J1F{22CPAHJ|RpoH?ng4anHR-7~;-Wo79u|7@R5!?=U)bS#}b z>Y2%WDDlKZhPqa~qS$$zV?cNF z`G6M3n?5rqeCpv1Q3aa>KJ$_Jrmg{K6LPti1vBol@6}8YLb(tHDwU#t7Mv7c((bSZ z<<_z32~u?Q>QlQnG-DpFPJU4RD44NYs@zu5ZGDFh-Q?y`v@~?J@3HsuUaiJvB>DkG zSAiL^9jsTeMOR;&v$@Z8mZ++uKph-}&kJFK>Md=Bt1^;#KX|)s5U)f_cV>RiU{jxP zee9>W0#==5)xvK{E9KnKV4~fUhmDp)>r^P}19-qT|~JWYWY(Z2C)CHO7}YLV(Ij4xDg)@O+0^v8}9yG}{LCLsk=GH72P ze~%>9z&+|3Djl|;>foC432v>qWN>)>x)WEDj1EOHr$|ip3P^zT>w!A%4+)w^Ax&RK-t0RV_=@T=|6W; zq{n~!D{8+Y6*a+!(!-1lyN+QWGQIq}?wrjr9S2UFNq} zW*?6TT`9RDhusIWODu_Z^VJ3{V0XSYuR=*{a8sB_B_@A;X3AM@Us-Z zyGO(BSqz3=7->W?3tRJTaj|DZSr7C!!=3y4f01(r^9j#M#`$ z#)RqDFUv39pdo9w#QCxlb>5BG+0JC)%|6g2e70*p^?8$xIimNfr5S9nVLE&k%B zFH~=fK?v5oO^AV(ANG8~E&BqmHNj+6bthpGP1<-KxCx~Kl~ele9s(zQbC%c}n&VAM zzU{E&$T&y;$}r}Z9c6LuNaR6uL{#bP{9xo@z706EzOvlsy4$3!P!t^vcu}L>YeT0( zt5IjO*Ug%Q;~{G3&bdL_gIGvf?d^x^Yu9(^SeX`MVVX*cK(-H-m zvaC5r%ezaOT|{V~s?5(}`s(Psci;94#^QVgb~-~3zI(1@DxZx2=Fhv7%%GMIo6^S< zieNv5M6|VXG|iQ7aNI}1{Io2Zc2J|c9zb5x?rO568;kOUSxf)AB7Vn*zj7=C0~tS| zL%O~%m`yar%JLJH*ldS!#+zVer7-SW;2>**IhkXJGNslUsZ1AqGvdh5bJe{2j2MaW z4ZDP8TrAop;7flb3Je%#ox{I@>L!i~CrL>|%F~~+)W$nqzc}>8)^;6{o}ib1^s$gY z%fra$0657jAI{f0y$;#R?br7aokZT@_nv1f5g~dXPL&29?x)Q zFHXs) z+9B=g#@NyB0R2%8Oy5dUQv7j@c#E5uMUmQj;0m54cBP>d3qnEe5RzQ+rie`8r zyq9Dj;b_7?bE-*DxHpJS@$^9r%JzDPpu03Kc7OvW(~0-uYnT&(UuQkjtI;SOK7XZa z=gmEJh~o3_h?;115G#9of0Xy2II;?$O0hKUP-N;IYuaJN5%j)oJ*OlbTmm9)csspY zOJD2%)>fW5(Md7UQe8n*&&K(AO8F|3wXuvcayeGxAxS@TEu`N%4lOa{CwZU&Nu@s$ zaC|_@{yT!|SKCkw(55juv_>@wxlD6lSx`fQo2yWPzEk5jd8S0)0n3Qn8%MED>m=)1 z4PqCOsDy5dAP&a%OlB&$Or_M#)Pj$LGIbnwqASGd9WwTB_CK0SVUC7o{SYu?em;Va z{9bw55YsRVH;LmmeE|Qo(2J5{d~mn{hH5Edwr^t}nV`Nxwdq)Ms8@OkuiFA4SP}ZC z@uCGY(njNU$#Pe`mBaXks?!@D4z~~8SZHCfmKY<MNB?5l_Y-`-2U5@=C7*W!=`6hdza32r$uIYMx>TN-k z8D+6Y(pXgqhqn!#y0-w9d>xe4#=dF9!SPgv*}F3EdXnE1lliiWyR|Dn!3jaAAnlb0 zFw8wiH9_wqH!wph)q$qW=Kym?6CHBS&u2MqOYd-bx}m$WP}%+E__VSSM|NiJm}J5F zYYequ8(b~dS5G$6n{YY(B-BQ6)05cSc*7y%Rdh#5IW67@w;_1N2Yb9Zo*3+hi-fcr zc}C)fc5>@3Qb>6nhL zPPZIl`gy`6m-KGxym9};NkPb?0_|4hqa$*^SeKyl@Qag>XyyjeKMHtD1plqwU_7=P zEwM)d@A2(aPsPLD#7XC=*)+t1ZF`;+`$@i6t8e-y#Z^^-*moL@I+M?k$ARTP6f*sK zj`$E_VW3MAQ=?37$J(wu$Q5?@AmI$AwJa)@O3|u*-F=I}ss(~LRe5l#AhF`^e&57J zvXYxa)0TOve9_crs+MEvgbkE+5~wNzfg%Z%rl|CTZBq!NIOVU`q6V)P`?_%5`zk_G z!C5taQJWik_G_AIh)Rxg0HkDWHr2MnSN!^oW_J-CHpLVA5|Xj3c?Xu}HW(|JaG^3b zJ*{$-OLz~MXlVm@<#`M47BrnkmCDhuVaCO)iGicFe1-3Xzj2)ymMzJ6D~)WDg@~-T z_gVomIL+UZ=_0A~w#iM71c(Mk=@};7nS|;RV&o~9Z$;PUsmwQc>gv0e{{q@oqs3Bf1f)F!vAyr zod*Bu{yRB*V$XjW!Q)hZjr|W?`p+l*PQISVp+~3KUoM literal 0 HcmV?d00001 diff --git "a/A19 - Kafka/pr\303\251sentationKAFKA/consumer.py" "b/A19 - Kafka/pr\303\251sentationKAFKA/consumer.py" new file mode 100644 index 0000000..57a0d5f --- /dev/null +++ "b/A19 - Kafka/pr\303\251sentationKAFKA/consumer.py" @@ -0,0 +1,23 @@ +# Importing necessary libraries +from kafka import KafkaConsumer +import json + +# Configuration for Kafka consumer +consumer_conf = { + # Set the bootstrap servers based on your Kafka cluster and host names + # For a single node (localhost) setup, use 'localhost:9092' + 'bootstrap_servers': ['localhost:9092'], + 'group_id': 'my_group', + 'auto_offset_reset': 'latest', + 'value_deserializer': lambda v: json.loads(v.decode('utf-8')) +} + +# Creating Kafka consumer instance +consumer = KafkaConsumer('my-topic-test', **consumer_conf) + +# Consuming messages from Kafka +for message in consumer: + print(f"Received message: {message.value}") + +# Closing the consumer +consumer.close() diff --git "a/A19 - Kafka/pr\303\251sentationKAFKA/producer.py" "b/A19 - Kafka/pr\303\251sentationKAFKA/producer.py" new file mode 100644 index 0000000..e8bf277 --- /dev/null +++ "b/A19 - Kafka/pr\303\251sentationKAFKA/producer.py" @@ -0,0 +1,30 @@ +# Importing necessary libraries +import json +from kafka import KafkaProducer +import time + +# Configuration for Kafka producer +producer_conf = { + # Set the bootstrap servers based on your Kafka cluster and host names + # For example, 'your-host-name:9092' + 'bootstrap_servers': ['localhost:9092'], + 'value_serializer': lambda v: json.dumps(v).encode('utf-8') +} + +# Creating Kafka producer instance +producer = KafkaProducer(**producer_conf) + +# Define the topic name +topic_name = 'my-topic-test' + +# Sending messages to Kafka +for i in range(100): + message = {'this is message #': i} + + producer.send(topic_name, value=message) + print(f'Message sent: {message}') + time.sleep(1) + +# Flush and close the producer +producer.flush() +producer.close()