update docker services definition, cosmetic changes in diagrams, new …

…results

.gitignore

@@ -31,7 +31,6 @@ build/
 ### VS Code ###
 .vscode/
 
-/openfda-consumer/openfda-consumer.log
-/openfda-producer/openfda-producer.log
-/mongodb/mongodb-data-2022-01-20.zip
-/mongodb/mongodb-data-2022-01-22.zip
+/openfda-consumer/**.log
+/openfda-producer/**.log
+/mongodb/mongodb-data**

pipeline-architecture-pl.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2022-01-27T15:10:22.590Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/16.4.0 Chrome/96.0.4664.110 Electron/16.0.7 Safari/537.36" etag="AujtaNadO9W64ynxdxRw" version="16.4.0" type="device"><diagram id="Futf8SIoppectOjmvB_h" name="Page-1">7Vxrc6u4Gf41nmk/OMNNGH+M7aQ727PbtDmd7vabDLJNjBEVOA759dUNkEB28Inj5CTOzubAqwvwXh89iAzc6ebpbwRmq99whJKBY0VPA3c2cBwbBGP6D5OUQjIeuUKwJHEkOzWC+/gZSaElpds4QrnWscA4KeJMF4Y4TVFYaDJICN7p3RY40a+awSXqCO5DmHSl/4mjYiWltj9uGn5B8XIlLx04I9GwgVVn+ST5CkZ4p4jcm4E7JRgX4mjzNEUJU16lFzHudk9rfWMEpUWfAQ/u8p/gHw7+96j08H+fx/mSTIeemOURJlv5wPJmi7LSAIqoQuQpJsUKL3EKk5tGOiF4m0aIXcaiZ02fbxhnVGhT4QMqilJaF24LTEWrYpPIVnFNdqG9zyZFOd6SEB14oMpHIFmi4kA/p7YAdV2EN6ggJR1HUAKL+FG/Dyh9aFn3a9RMD6Smj9B65SOK2rPtPInX2wc0cPyE3vhkTujRkh1FMEVGq3yDcxptmiZhEi9TehxSxSFCBY+IFDF152vZsImjSBgN5fEznPP5mNkyHKcFf1AwGYBZbRU2AXoaGGJNDm48XLXXfm/rKl3OPrSuLDCS15DpQvpnb7PIye/Ywygzu9qkw/YMeLHIqbe0zVrfosnSeP7A8g196gSGaEXzCiKtlOR9LzN5yzhD6SKCw4zgaBtSy8geU5wWME6ZqZrgowNRuEpxgpfyju8zEqdL2jRh6cKh92L9Ch9h1X2G8pDEWRHjVPTfxCHBOSKPcShdJxG+Ii4iPGyBuY4ap/L/t8VVwzDnwXpNO9h+9tQ0Vl5Z+2kjAPKJHVA1Up2Jq+hjovixFgExsFEEHc0u68MNTR6TdJ5n9eSNVmgnMFOuos7YCSDTZY/VgX1IB3KWkN4b0WYZOO6Nz/4zjQW64V7UWvsxnZfSjK1EhExLSgah9Yh75wo9wSVznIl43Bng6QCRmDo/d0zZ464S0Uknu1VcoPsM8mS8o7Vfz0OL+AlV1dzWSgQ74844geF6yeVTqbcUp4iPTZJprUrXca+d2VhquZrSkedKvwX/MeXAvCB4jZSu1s1ods1SHH0ceBPFhbwvPQdOrCtqkqnF/2feNqUym51dOcAgNMlGulB0snWZsaNhRqt1aZ6k22l8b9LuFNO9edi3tGzpuvJ81+Afx5KylQJ97JF1dL2kpzKRHlE+gf0hUEtOQUZxzTBm47tcdhuzx+PTojSqeoQJzPM4FELZ5cTgx+kJfsDbgJ/jSuIaLtZ1BetZB69pwlnRAmP9XR3cLX8ozxmud6z5Nr/Uv69a/1SMv7/+hWUS00RA3KYC2oBri1BvFj5FI4+lvpeq3lyklG/zSqDnmEvZ+8hlr65zVd0DoFv3PFPdc85V96yPV/eaqqaWPvvspQ/0LH2jj1D6qtUgXUjn281lNXiphmeohiroO2I16F9Wg1+6LPrOR18OOu4HroInLHGjniXO3mPQV5KottXyBGC1DCzuTA5rbEyVBEulm4yr/hfyPavlM2LKnkxtTy8CHXC1K/PnreDkrQimZbjqVhPapxxMwSBwIIuJMo3XsdEdfzqyXoTV3rxA85HnBXpqGJyErKeZQ5t26IyAPsnb8/UbnC7xkbDsNzZmNtkLwyIcUrDHHzWC1CAwv6CxL4vG1HXI8dzEhYf4xIDLapXZwAC43pWHqN7KnhdwUfWR8g82/gpUp38OKpqdnsyetLNSPbtTFjCNnU4Cy4K+sOyVqOx1Ow78DrrJyDNSsYvYZtDOz798/343YI5120371r9u7r9/ErDz4tYERwtK5yRQx9Xxk38elPM73Cg81O3sugt/bp6ojWhwUvF9mRdosx/UkC1jomBErcHwjIUeEUf374Nx+LNp1XpuKNU9cU8FmH4SNBPyn3dBM8FBNKPTPS9gl6PASTANvLF/DBiBJKxAjGXCJiPXHwVOF5scTkqTGhh04UZbNjo9qhkZYI0J6ohLnBL/nADveDrcsa2PBne87m6998A/H2+7gd13s6W9h0o4rny+mkdyrdYO2SMHHEs8/dhLoYQqZBjBfDXHkERHchC31Opr8UpoRqcQR3cldbZ0bwnfoTnTapYlFI6JAnFhJb4mK6FRx/tpic3Tku32vwq9K+o8E7bLHhHFQ8/JT9h2cD2dXfiJs/ETQdBKkqYXQsaC7b+6YB+3vNmyxe1kWhZrvEvjdTfl3tHRBxLjZeXy+VcuGjTpZjz1JbaSX9g5K9F1yuIJL9cTXitRWYF7OwLGfOO7NpgeSFmGvJtx13VM+exHs9aHWGq8VdYat6CdZ0haxrfYwZlWGa5pU7NwaupXqTEAZTCx4CPL+V8sTtQxxKcc/ZUdcraOh+sCbuKkFGNqyER9zfVYiUbJI2KkXKdFn6Qd81qbuEvWmGKy4RyS0ryTumXtnrhP3pjQtQ8iQxZCbG+TaTw1fjGU3s6apcNrzextSSqnt5Rb440FgWm+oJNW0zPCU3bYYRLpV1eHz2swMmzp3GG7BKWuW8eN5qM4pwVLaj1Ok1i58CLBsFBvqJ0En2EW5/VLaANFq+Q+4Slq8lNyGdPBy9jsBLE2au2kBE432AJDrLnHh1rP2HI6sbXmW+8+z/d2L3AswfhqPPbHVMUj+tvReWfvkOL3LJVtoE0x9E5AXvc0ZpeO2eA0LjAxWnNXrgl7z/G5DGpbgFm0+dEt6oy4vW3gBh7/3du+lT3H5zKn141No60+xnauHgSb8p7SsvQ3lVe+Pxq879tK77ycXSdxBOMW3dteFO7ZRHYyd/N/bu86uSuc3sItQt+3QC8LH0vvtlGGMzrM7gb+6/pXz3EWMvhhm5UMD6e4QHOMa+6iJw38qxhO235vTWAgOawdmg/Zm9qIo1M6Dobc/qxQbbJtIT42oPddcPB8YYe/KDusJYwLO3xhh02712w9a/bevPZ6crgnAHANePMUwVvsMHkuu9mCrt/LAqYx3BuO5nX6Gy1KGDjJm09xdPc2BJEUncA12gXVtru+4Rtcw3srWqDavv7jlBs0c26DPuwbXbRiwRjv4dZ6ElDsKSrSyRsrBBS7uKFKtP9YF6AaY1L+l6jqs0qDgOuQSmbsmE0OmJYA2w/4Ql+77ls5yQ9N4zTTCEPVLcr3GYDbjbbwwAP8adkZ/zzDAVOa9wDf5giajzSqWYWR61kbYoBNwKOQtdn8VEQiO6/YR9CORtYo45E1NhHJGiwuq6CtuCRHLCxPA26f6llFqJm0qdIHBpV2tF0PrRXdBCdg4Vn3DCxlmlKxwdi6qqgE9rvpBPOm11KZtW1cflpbWBXqfif7dRy0H8X5s5M6vk7q6BwbEJSP5PDc9rKqB6lzgs9oen6p3iV14JKg5SEa+1N8SfWigS2DSdW4Oh4CNX9BcNCKHPfm/w==</diagram></mxfile>

pipeline-architecture.drawio

@@ -1 +1 @@
-<mxfile host="Electron" modified="2022-01-26T11:53:51.006Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/16.4.0 Chrome/96.0.4664.110 Electron/16.0.7 Safari/537.36" etag="EPShmHeMFiArLR5kucbT" version="16.4.0" type="device"><diagram id="Futf8SIoppectOjmvB_h" name="Page-1">7VtZd6M2FP41Pqd9cA47+NFL0jntTJs26enyJoOMNcGICrzNr+8VEosAb1PbSTN2ThK42u/2fRK4Z44Xmx8YSuafaICjnqEFm5456RmGbnsD+MclWyEZuKYQhIwEslIleCJfsBRqUrokAU6VihmlUUYSVejTOMZ+psgQY3StVpvRSB01QSFuCZ58FLWlf5Agm0up7gyqgg+YhHM5tGe4omCBispyJekcBXRdE5n3PXPMKM3E1WIzxhFXXqEX0e5hR2k5MYbj7JgGn83wV/sXg/7ubi3695dBGrJx3xK9rFC0lAuWk822hQZwAAqRt5RlcxrSGEX3lXTE6DIOMB9Gg7uqzkdKExDqIPyMs2wrrYuWGQXRPFtEslSMyQfauTYpSumS+XjPggofQSzE2Z56RmkBcF1MFzhjW2jHcIQyslLngaQPhWW9Ss1wITV9gtYLH6mpPVlOI5LOwdUNJ4KJj6YMrkJ+hVeghbTTLh/RFOJN0SWKSBjDtQ+NMAPBCrOMgEMPZcGCBIEwG07JFzTN++OGSyjh40Dn9qhnT0q78A7wptcRbbJx5eN1i+32t7baZe997U6zXTmGTBjSQ482jOz8kS+m1rOpdNpv9kBnsxT8pWnYcopdtqbTzzzjwKoj5OM5ZBbMGknJet4mcso0wfEsQP2E0WDpg2VkjTGNM0Ribqoq/KAh9ucxjWgoZ/yUMBKHUDTiCcOAuWg/ohUqqk9w6jOSZITGov6C+IymmK0IhEteKRK+IgYRPjajuY4qp3L+WdKioJ/m4TqECrqTbKrCwi9LT60EtlyxYReFoDMxitomIKtSZIuGlSKgNR/WQQtIH6N4miZl55VWoJI9qY1S77EVQl3DnqoDfZ8OZC8+zI0pvfQM897hP11tbdVwB7XWXKZxKNHotYiQiamWQQCRcu+c4w0KueOMxHIndp4OMCPg/LljyhqPhQg6Ha3nJMNPCcrT8RrQX81DM7LBBZ7rCkjwu9wZR8h/CXP5WOotpjHO20bRuFSlaZhDYzKQWi66NOR9rd4s/3TlwDRj9AXXqmr37mTIUxwsB90HJJPzUnPgSLsDk4y1/Jd72xhkOr+7M+wOYZfMVYWikq7KOit29Kg1hs6TdDON70zaLTjdmYcdTcmWpinv1xUDMjQpm9fIj+5qJyMm3MpEegKA6sZ74y3GkbzFvgxvOQ3LXtDspYSeIwFsCJliDsig/VRv3MYtnKackhvadJnegOtbBa46Pd8NXP42IhDBzKygS7dzbTHwZuFTEHk8Zx2Cq6nIBR+nhUBNDje8est4VQJUAVi23QYsqwuwjGsB1uA18AnUx7Z/8vZ3llsK/gIB32npbiGZbOQY4m5bv3us0cDKXMdg3kEsc/8jlnVv9VxD3evZWsPGAotlq8rMQ8bQtlZNhtbx41iN85fT6sOFmMFFd5+wcU+Xi9vu8wbiVwDxOlc9Yffp3Haf3zSaO000f3PbzyKTXxfNUwDUbMifaXDnilCaEr8QP5DoApvSywC0rmmqgVsILZhDC6EPg+KR5rNbpwf/LHG+ND4WzCMMGQ5Rht/3Ybxw451xCPFvWZ4air1zHMbbKhGy1PaXP4pf0DikJzKgT7zNZLST8QTUB16VrzJAYAuU3ojPN0t86nnz9NOL20nFO+Y2egP6vA5u86onFUU6frWTCrt+TlEdSlzkiOIgA5Lwd/AoY4fVr/Q6gdPiMwyjoONVgg/Pz48Fy2nmee23+6fnd0JsDr5mYChRaJyF1pgqV3KuQ2t+RovaGc/DZNjmO0/bNMOL3dSFLfnRDgrABJy1aAXHfR0mky9IweRpByAfyW4KWvQ/4Sx+/nkVzuLt5Szq+ckBhnISBfHGnjVwTqEciPkFVdG6GIhrOq5ntBnI/kw0KuG/TSqaMvf83MXtIC9dhEYMcU6WcwZWY6mkRteuR2q+7hg8gizfD1A6n1LEghO3gg8RSl/EIfgEuhBXj1vgPPHOHLvGU77MJIkAJEUE3zaH3+bmUD/upazFJuRvVN/51h04z4i/yYxZzUOvuU3UdW84nty2iVfbJnqemlFN89gH2s6Vton623hz/MgzcRwHRSXp4SCR5WfeMermsVvGC+0ZT9s0LNPqYXCFk7CjTveg2W0/8P73A3r94UAbpurPWmugwO85rypjOkepVEWpBrponvng2p0g4Zi6Pd6DMx1gmeSua3SB0NdCzZsg8JeCmkHjYZzVwd07H7Z6V0IaU+9AGuHU4FdxZwDKYOLBx8Lpd1p+5sVpeu3qe36ZH3zl4TpDCxJtRZuS54KvmRZHHBytMD/fapWonTRjXikTs+SFMWULFKnFa6lbXm6JeeaFEeAfZn0eQvwVnK72YPysL72dF0uHV4r5k4ZYdq/VppYXZgzF6Qw6LbqPcVlhTVmgjl5vPi0ZZL+hc4O/gyd13biuNB+QFFBKap3EEakNPIsoyuoTaibBNKOMf1+qPK/albrB7YSn1JNfLZdxHRwm1GeINbfxZoNttIPN64g1+/RQOzK2jFZsyTfE3s/30A4cXXiDu8HAGQDYuPDXUM9wLWuP5ne83aDbShd96wwHwUdas/21whXB65xTpsvo3dlO10xuvOqjGs9wctPqtulZ+d+vwavq+7nCStW3nM37fwE=</diagram></mxfile>
+<mxfile host="Electron" modified="2022-01-27T11:23:13.324Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/16.4.0 Chrome/96.0.4664.110 Electron/16.0.7 Safari/537.36" etag="y2LuU_CcWZo7zXcExnUK" version="16.4.0" type="device"><diagram id="Futf8SIoppectOjmvB_h" name="Page-1">7Vxbc6M4Fv41rtp9cIo7+DG2k52a7ZnNTnprL28yyJgORozAsd2/fo+EMAiEAx234yR2VydwdD/XT0dyRuZsvfsbRenqNxLgeGRowW5kzkeGodveBH4xyr6gTFyzIIQ0CkSlivAYfceCqAnqJgpwJlXMCYnzKJWJPkkS7OcSDVFKtnK1JYnlUVMU4hbh0Udxm/rvKMhXgqo7k6rgFxyFKzG0Z7hFwRqVlcVKshUKyLZGMu9G5owSkhdP690Mx4x5JV+KdvcdpYeJUZzkfRp8M8N/2v8wyL/cvUX+932ShXQ2topenlG8EQsWk833JQdwAAwRr4TmKxKSBMV3FXVKySYJMBtGg7eqzhdCUiDqQPyG83wvpIs2OQHSKl/HorQYkw3UuTZBysiG+vjIgkodQTTE+ZF6xkECoLqYrHFO99CO4hjl0bM8DyR0KDzUq9gMD4LTA7he6kiN7elmEUfZClTdcGKY+HRB4SlkTwHKkVIqX9ACrE3iJIqjMIFnHxiHKRCeMc0jUOdbUbCOgqAQGs6i72jB+2NiS0mU5Hyh9nRkzw9SYR3g3Uhha6JxpeF1eXVrW5vpovexdqPZrhhDuAuhn73FIjp/YIup9WxKnY6bPZDlMgNtaYr1MEWVpMniG/M3sOoY+XgFfgXThkuyvu5TMWWS4mQZoHFKSbDxQTKixowkOYoSJqrK+KAh9lcJiUkoZvyY0igJoWjK3IUBc9F+Rc+orD7HmU+jNI9IUtRfRz4lGabPERgLrxQXulIMUmjYknAeVUrl/LkhZcE448Z6CxV0J91VhaVWHvS0IthixYZdFgLPilHkNkH0fCDZRcOKEdCaDeugNTiPabLI0kPnFVegkj2vjVLvsWVAqmGH8kA/xgPRiw9zo1IvI8O8c9g/VVtbFtyLXGsu03jJzeg1ixBuqeZBIB5x7VzhHQqZ4kyL5c5t7g4wjUD5uWKKGg8lCTqdbldRjh9TxJ3xFmK/7IeW0Q6X0VyXQgR748o4Rf5TyOkzwbeEJJi3jePZgZWmYd4a84ngctmlId5r9Zb8o/KBWU7JE65V1e7c+S1zcbAcdBdEuZiX7AOn2g2IZKbx/0zbZkDT2duNYSuIKporE4tKukxTVlT0qDWG5k666cY7nXYrmHb6YUeTvKVpivdthX8MTdBWNeiju9rgeAmvwpEOCJ+2fhGoJQOQkd8yjFnpLqfdR2x5vFucBGUNP0ZZFvkFUVQ5MfgxeoIf++eAn2Eh8Qktnw4RrGccvAWHs4IAo/293rgd/nCWMVxvaItNdo1/nzX+1TF+d/zz93EEjoCaVQTUbc4tCtpc6BRYHnN9L0W9ReFSvixKguxjrmHvksPeIc6Vcc+223HPUsU941xxT7u8uFdFtXro088e+uyeoc+9hNBX7gZhI51t1tfd4DUaniEa1kHfgN2gc90Nfuqw6BiXvh00zAuOgicMcW7PEKd3CPSVSVRda2iCrTUEXMxMNKtkDExC+1o1YVdHBtLlgRxLa+hM0WXPTG1PLbJb4OrPDRba+u4z74WNdBo5OBfL8mQ7H50k8w5uQOp2bJQHgOdLvq9JEpKBGOs31mY+7cRUAfEBufGlsvOZBcqu0OrTQqv6pmJ4ouGaVPjA6ElrxExPgZ7eNKlQHrGeFz0B++j+P6z9jV2+/ndU5szhZb6T3vb1t4fabqSS00kwltcXY70SYr3u+oDTgioUo0BxdeCXr18fRkyT7tt+Xvvj7vHrB0E3L14sMCQrNE6CbUwZMDnngTW/o3Uti3Q/v23jnbsdyAisEciP+yzH624UQzcsj4QCkAYDMBp+xhybvw2o4WuTwvNCEZt7Ap0SIb0T+OLzz5vAF+8ofJGTNS+AlUFoxJt51sQZgj4Q9UvUoqnAiGs6rme0wchxpzQ9IIE2vmjS3NPDGFeBY1TYphjilIDnBADHkvGNrl0avrHad+3eAvBc5KGJ3ve2pN6RPhgWQYcngpq5R61xxXVgg6GZox871YmBIeMAZasFQTQYmHe4B6k/FWc6c+iieHrYg74lnVF8ixeMq2kaAyIrYsQ1E/E5MxFS7rc7FbHehey6/o1v3YDyTNk1eUxrGnrOnISue7ez+TUncbachOc1nKTqREcZs51Xx+xhOxwIfONNVp2PV772AZod8YjXXcvH37VImKTt6urHzzXHwt5ZbD74Ku7pMtnTNTyU5pn3rq10NI6p27MjvkrhcFOuuobKkf2ou7qIbcbPcleTBqazFN5Kef7snWmHYaquIxdKDXqVKA1QGBMzPhou/qLxJB2DerWnv7JHnqnj5rpE6yjeF20OWAl0zbRYbMbxM2YJuVaJ3EnT5qWyYpasMCF0zfNHteKt4C0rt4p58sIY9j2YjpkJsVtJqvYg/HwstJ0VC4WXitnRSCK612pT44U5RUm2hE7L7hN8qLAlNJBHrzdfHFDIuMFzg93vE7xuPFecD6IMIpXgepTEUW3gZUxQXp9Q0wlmOaG4TKZ1O25QukJP6q6v5skYB16GZCewNLdxA9I22qbmKSzNHG5oPS3LaFmWuDL3Ub4m90JyxZvcTCbOBDjswk9DTjhbx/jesUHWbamLsXWCrHVPWbbzMM8R3hbOrCFKf4Vonn0wYeqaxaRZfWRpGi6XtW6bnsV/9pZtKcvJuURptc1SKat3k1KrnUZqmnweeeM47uhtzySt82bpWk7DmzRyvM1tYMe9r5Opm/O+tevkqnB6CTey+I5m95Lw0IRuE2AY7vF8rue8rn65jrOkf79t0j0DwgnJ8YKQp7JGz8Tvr0VzKPu90YEiu6Ft8WLMjmcDDkuhHfK5/FmgWqebvPh+AMw756j5mg/+pPlgyWFc88HXfLDqjlrjunXvK2qvTwf3BACmAm92WfhJzfkHe4GNTYchq/c74qJ5kwyy3sR51ukT1HmCn7QzYggpq77CI9uYwpIF6QT62YzqqjsGtkI/f0A9e+qj19JHIcHipp2GwpDiEOVYCVzf8xbWkbewcjbBtnmpyFaYTRDZzz1UfxCrAGjVnxUz7/4P</diagram></mxfile>

pipeline-cluster.yml

@@ -1,8 +1,7 @@
 version: "3.8"
 
 services:
-
-  #kafka cluster
+  #kafka and zookeeper cluster
   zookeeper-1:
     image: confluentinc/cp-zookeeper:6.2.2
     container_name: zookeeper-1
@@ -167,11 +166,11 @@ services:
       KAFKA_NUMBER_OF_TOPIC_PARTITIONS: 8
       OPENFDA_API_BATCH_SIZE: 1000
       OPENFDA_API_BATCH_READ_DELAY: 1000
-      OPENFDA_API_SYNC_DATE_FROM: "2000-01-01"
+      OPENFDA_API_SYNC_DATE_FROM: "2021-01-01"
       OPENFDA_API_SYNC_DATE_TO: "2022-01-01"
       JAVA_TOOL_OPTIONS: -Xmx1G -Xms1G
     ports:
-      - "8888:8080"
+      - "7777:8080"
 
   # consumer
   openfda-consumer:
@@ -203,8 +202,22 @@ services:
       - openfda-producer
       - openfda-consumer
     environment:
-      MONGO_URI: mongodb://root:openfda@mongodb:27017/openfda_test?authSource=admin
-      MONGO_DB: openfda_test
+      MONGO_URI: mongodb://root:openfda@mongodb-primary:27017,mongodb-secondary:27017,mongodb-arbiter:27017/openfda?authSource=admin&replicaSet=replicaset
+      MONGO_DB: openfda
       GRAPH_REFRESH_INTERVAL: 10000
     ports:
-      - "8050:8050"
+      - "8050:8050"
+
+  # jupyter notebook
+  jupyter-notebook:
+    image: jupyter/base-notebook:latest
+    container_name: jupyter-notebook
+    hostname: jupyter-notebook
+    depends_on:
+      - openfda-producer
+      - openfda-consumer
+    environment:
+      JUPYTER_ENABLE_LAB: "yes"
+    command: "start-notebook.sh --NotebookApp.token='openfda' --NotebookApp.password='openfda'"
+    ports:
+      - "8888:8888"