From 63d84a0dcc19633100c4b712546e11e9c8cffd15 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 09:52:14 +0200 Subject: [PATCH 01/50] changes server and worker --- Dockerfile | 2 +- coverage/lcov.info | 657 +++++++++++++++++++++ docker-compose.example.yml | 3 +- worker.py => server.py | 2 +- {celery_app => worker}/__init__.py | 0 {celery_app => worker}/tasks.py | 4 +- {celery_app => worker}/utils/__init__.py | 0 {celery_app => worker}/utils/fire_event.py | 0 celery_app/server.py => worker/worker.py | 7 +- 9 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 coverage/lcov.info rename worker.py => server.py (97%) rename {celery_app => worker}/__init__.py (100%) rename {celery_app => worker}/tasks.py (98%) rename {celery_app => worker}/utils/__init__.py (100%) rename {celery_app => worker}/utils/fire_event.py (100%) rename celery_app/server.py => worker/worker.py (57%) diff --git a/Dockerfile b/Dockerfile index a41cfbe..fcb40af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ RUN chmod +x docker-entrypoint.sh CMD ["./docker-entrypoint.sh"] FROM base AS prod -CMD ["python3", "-m", "celery", "-A", "celery_app.server", "worker", "-l", "INFO"] +CMD ["celery", "-A", "tasks", "worker", "-l", "INFO"] diff --git a/coverage/lcov.info b/coverage/lcov.info new file mode 100644 index 0000000..3d54539 --- /dev/null +++ b/coverage/lcov.info @@ -0,0 +1,657 @@ +TN: +SF:bot/__init__.py +DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg +LF:0 +LH:0 +end_of_record +TN: +SF:bot/retrievers/__init__.py +DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg +LF:0 +LH:0 +end_of_record +TN: +SF:bot/retrievers/custom_retriever.py +DA:1,1,1+9wdnNK6ERHmSA06tdyPQ +DA:6,1,RDWQRIzCg9gLwFgbUhG0ZQ +DA:7,1,97oNYmi8xQccedqGxd27UQ +DA:8,1,Epduv66JxgjoxUaCT3nS2g +DA:11,1,s6T4UohrMBIsrcI68IgiqQ +DA:12,1,XJc56GRwdNlz7Pfca4ta2w +DA:15,1,FhljCPNPys8XCLWkwhX6tQ +DA:16,1,oQZL7x34ZmpY26UrXDNFmQ +DA:19,0,Qa1wQ2wVs41AUQxepYAgAQ +DA:22,0,mRVOQmWFDi50EtfArtMGKw +DA:23,0,ziMwbdP43Yl2rkv+Iw0+Vg +DA:27,0,/pJeNEIa+r0HHLINHDruGg +DA:28,0,GuHBXE7dyAR1Y0ynfbXBjg +DA:31,0,braB6j7cvq7N3Zid4KaqrQ +DA:32,0,5YC65MS1vK1RBYkxZkINoQ +DA:36,0,XlCerKuXPRvHHiX6G4l0nQ +DA:37,0,+4WCHe8YiFhx16gV8PHaiw +DA:38,0,FB660ofQJW2XNi3LECQSFw +DA:41,0,zpXzPHqwITv7OgzCdXjxQQ +DA:42,0,hqLZyd6T1Z9Dbx9cHMDy5g +DA:43,0,mj69/XSznAeubuOVDDQgCA +DA:47,0,36hUmZ4dEI4TxxoXjrxRWg +DA:48,0,GmJAjoIgF9rNAoVVXoryvQ +DA:49,0,4HIKapx7G/1BQUJ4xQuPlA +DA:50,0,qay3E+cJPb5xSdPWZrpBew +DA:51,0,v50IHDAPJiYZpVFrAPWUCQ +DA:52,0,ut8wnco13WKIzqf4F3XevQ +DA:54,0,xGgKt1DFNSwI4OqNYuCmqQ +DA:55,0,XPrAAc6YApLYKhPB2ugc5w +DA:57,0,bsKhhD3OVsNd7qpCNCVshg +DA:59,0,dSnd2LGkYzu2vL0jseg1aA +LF:30 +LH:7 +end_of_record +TN: +SF:bot/retrievers/forum_summary_retriever.py +DA:1,1,l4yWWF9Yn1gF6iRCnynHUg +DA:2,1,qgap8v00S1Ud7dKp+nEVgg +DA:3,1,ZwFry2q3ybNGgdFz5V3nNw +DA:4,1,S6WdzE5Nj4jcMIr2/RIeEA +DA:7,1,JZoyIbA0/Up5YJXsqDUYFQ +DA:8,1,4LgMgWBGno1n5MNJ8WRFsQ +DA:18,1,C1mbXdO4LpybcjJNbDJYXQ +DA:20,1,f+GuTgPLC7rYJZEnF8KkZg +DA:55,1,Rrf/KyYVoV7ZZGZVZ81NWQ +DA:57,1,/MYzux0mSuxPnCfDB/QApg +DA:64,1,x32FHFF9bNjommgpDHkbdQ +DA:66,1,HDI5TfzjszdVNEtxufbnyg +DA:101,1,exJXoRLYgNNdu03/ShoHrQ +DA:102,1,kUdJgUS+mcODFLg7Fk+OHw +DA:103,1,+f2nVTBEmuKiag075WVHkA +DA:105,1,osmxeRTnhC30Fq8zkebgGw +DA:106,1,0zgtOd30L7H3j7fx+w3Mdg +DA:112,1,K+yjYcF/QNzhfYmE7uZPAw +DA:113,1,9nMOpyz632hMtLnVoiQf0w +DA:115,1,dDsC6FL/ONgaG0AYHRlC5w +DA:117,1,x32FHFF9bNjommgpDHkbdQ +LF:21 +LH:21 +end_of_record +TN: +SF:bot/retrievers/process_dates.py +DA:1,1,EZEzNmL93PY96zsSuI0PfQ +DA:2,1,w4wzKKlLBqr27YH2UrIkBQ +DA:4,1,KA9tLtxJCmXGWOalg5c/zA +DA:7,1,+QD2wyYZcsC+Xvi5rjfg1w +DA:26,1,bFij5SG38wK0MWxmbT82Rw +DA:27,1,iBjZwelGAT6u9wcCT5RkQw +DA:28,1,K1+sxVnEq9jO8BqcJwVAZw +DA:29,1,pf92umTBWmtiEB1EpuczhA +DA:31,1,2uLiygcEABFP5hS6AeQwHA +DA:34,1,tYltTNWket1uI7UNICnqmw +DA:35,1,RwimGdhYyabPJL4e+Fvqnw +DA:36,1,LTTh/+NrsNUrIQkwnwP3hA +DA:37,1,Da4uV101pljkvITvb/orrw +DA:39,1,p3JK0mYGwzpZUPpV9FxrSg +DA:41,1,7mawqEbVi5mg0yRWqaqupg +LF:15 +LH:15 +end_of_record +TN: +SF:bot/retrievers/retrieve_similar_nodes.py +DA:1,1,usL6GpKXVxVT8icw7bX7uQ +DA:2,1,nuxjJx40cdYGSgrgJ50F1A +DA:4,1,KA9tLtxJCmXGWOalg5c/zA +DA:5,1,vMmpgxZJRuue6rFB1P0BSg +DA:6,1,qgap8v00S1Ud7dKp+nEVgg +DA:7,1,ZwFry2q3ybNGgdFz5V3nNw +DA:8,1,XJc56GRwdNlz7Pfca4ta2w +DA:9,1,IDxH3gjD3VSVrI4xLmrtQQ +DA:10,1,YQAIr2ESlBeDuRf0TeEF5Q +DA:11,1,2PHsUT6zqyYmFXFvARFHDg +DA:12,1,S6WdzE5Nj4jcMIr2/RIeEA +DA:15,1,IPFoc7JFmfs5ZGCvDdLKww +DA:16,1,mqoJzotpXwlnL7jkyBBK1w +DA:18,1,4LgMgWBGno1n5MNJ8WRFsQ +DA:25,1,GeCb9dZfUzto8bJsxLM/OA +DA:26,1,BOdqpSj22HMpl9h/3UQEnQ +DA:27,1,EEPdD4zi1M7G+hCfGc7kFw +DA:29,1,d+ncuJjKr272GS184mU5Mw +DA:64,1,4lzkDJLlQXSyBjO/ZBZ0tg +DA:65,1,R2Ju8xaiGLrApGDnnYA2Cw +DA:66,1,sXjQBJruC0xTKR4b+u8JIA +DA:67,1,pWP8puMbrWzJ9jyYSNijYg +DA:70,1,1DaXT/QWnDNNMkOiYOOCAg +DA:72,1,M57N117IoUfJdZ4DKcJp9w +DA:73,1,ACylfUs9k82fbh1JUVpkFw +DA:88,1,i9tYTGC/B+qp4Thp+0XtPA +DA:89,1,eJXGGDxmpnTlPApZ05Mm8w +DA:90,1,bkIDFrOGWBbeePkt6w8EJQ +DA:91,1,IDhEFBr50D3yQ1PgRwIPoQ +DA:95,1,DVM8lfdtc7XvpdnChfyzZA +DA:108,1,NYgySTsjOvX0beJFX6yMkQ +DA:109,1,87A78EES6C0LtBxGYDZtBg +DA:111,1,Dza3guFRyWWcXuaktAvUUA +DA:112,1,42k3Krouu/7c6z6xkNm7jg +DA:113,1,74cSsiTs5tJX4LFRiR9Z/A +DA:114,1,NcFv4vwj+pjug1e2Bo890w +DA:115,1,Bp6QbYpwe4PVOg8lbJA/hA +DA:116,1,jrIPKZt5zHATQ3k7gB0zMg +DA:118,1,3X+u/Gx10tHDgUlKjZ9leg +DA:119,1,enK82Bd/RAeIx81NTDQ/7w +DA:124,1,AEFyjNRKgOrooYQ/nYO/gg +DA:127,1,unkkFkVkr+wa6RT5uE8c1w +DA:132,1,HJEWWSze80ShpmLhx/FNmw +DA:137,1,SkkggBMbU32rJ3pNviGxAA +DA:142,1,XDQoFg/N3OcYqb/J4NJzUg +DA:143,1,QAy+2Mz9GQWA6hn/DfyOnw +DA:156,1,Ap+b3WevKixvNInY8BOstQ +DA:158,1,M4Wpq7+IHVDAhVO1Q+IA+w +DA:160,1,tzZodAsv3aFszptj+1OcnQ +DA:161,1,r+V3a23Womuti9fftxA7Lg +DA:165,1,e8upNNRAcwfVSEfPvxiucg +DA:167,1,FIGBJFFS77Q/1c40b8di/A +DA:168,1,NdO+Zxe21pmF6dXu3zWdMw +DA:170,1,UkwhkGz1XXrgDOp60bursg +DA:171,1,Cj+UwANUqSKqiqpqgnF9hA +DA:173,1,X1yR/f77a2wYrip3qWXYBA +DA:182,1,k9pTjjD2vJTvE7PyzhXJPw +DA:183,1,72SNpwVsw/o9etOCZ3VLaA +DA:184,1,5M8S6frycaNS3uQnlnMr2g +DA:186,1,3WsKRpW8oQhkQv0F14ak/w +DA:190,1,mUbbKvPqWHojLjV0v95bwg +DA:191,1,DdHNUt5ovAV1XQYsiT8AiQ +DA:192,1,qay3E+cJPb5xSdPWZrpBew +DA:193,1,v50IHDAPJiYZpVFrAPWUCQ +DA:194,1,siWxFuIgfjXgP30tTFf/aQ +DA:198,1,xGgKt1DFNSwI4OqNYuCmqQ +DA:199,1,XPrAAc6YApLYKhPB2ugc5w +DA:200,1,Rx8CNN7oYnLYIPSkwQ5G2w +DA:202,1,diWaG+5ccHGPOYFR5G1hZw +DA:68,0,H8roOaxT/dA+ek63CQkNsg +DA:121,0,genHDsC84NGAm0Lr+up+qQ +DA:145,0,I/UvgUIdm1YWL558qpxsAw +DA:154,0,m+4mRvWfRixXNFnr5PmwsQ +LF:72 +LH:68 +end_of_record +TN: +SF:bot/retrievers/summary_retriever_base.py +DA:1,1,3H1a4ALTm6+CsKOSplE0HQ +DA:2,1,qgap8v00S1Ud7dKp+nEVgg +DA:3,1,RdpxTAbMfKRSNi6jiLRhow +DA:4,1,S6WdzE5Nj4jcMIr2/RIeEA +DA:5,1,r/vTV1phZQGHkBIJdSWs9w +DA:8,1,uUFp+g6+u1KGwCpCapD0RA +DA:9,1,4LgMgWBGno1n5MNJ8WRFsQ +DA:36,1,+Cv21TkBoI5AI2DIkaXSUQ +DA:37,1,IMKXrfpxE/NMfZh6Hz035w +DA:39,1,giUPm3pCLbXkU0PTyH+bFw +DA:55,1,anaPnxi7G3x+uzrWDKUlIQ +DA:57,1,rosmnQjUb4d/mw28S/QFyA +DA:59,1,Gh3uqVxTsog8Q14gAbyV5g +DA:60,1,MTyrxoRbX0cdVfuQK+cdog +DA:62,1,5M8S6frycaNS3uQnlnMr2g +DA:64,1,8oWHAROlVU38/+SOWIYZ3g +DA:74,0,aTZE5skweNKnLAkkNEpaQw +DA:80,0,6m2c6rMWrwACEmomupZfAQ +DA:81,0,ItkzfojqvwK4mDljHSDeZQ +LF:19 +LH:16 +end_of_record +TN: +SF:bot/retrievers/utils/__init__.py +DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg +LF:0 +LH:0 +end_of_record +TN: +SF:bot/retrievers/utils/load_hyperparams.py +DA:1,1,7Z9Lj4ed27Wf2hBX6jooEA +DA:3,1,oBn3m+aX07XSXyLZ7hhJvQ +DA:6,1,pALyYLvuXgvTjbXn92fkbQ +DA:21,1,uMN5/uIZZnp9nEAOplwhrg +DA:23,1,E0RQB+VgBEJ4QKNtxT2ZCQ +DA:24,1,obtvc0wXNzMHpam/R9pARQ +DA:25,1,xhBQ4Bxy/t+hksYxRkBLMg +DA:27,1,EevRcw2hn5gowSBZyczEKg +DA:28,1,oBoY/n2VDmZk3l8eOirLDg +DA:29,1,EsZQAiXrpsXDNrw17z9J5w +DA:30,1,HZ54ZIWn+eFEat0+ZEm5cg +DA:31,1,tl0+/G75a26GKwswbSpW7Q +DA:32,1,rGIsTu9A/YXl9CkiXRexpg +DA:34,1,zw4+yRAywnofajwDMr1yLQ +LF:14 +LH:14 +end_of_record +TN: +SF:utils/__init__.py +DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg +LF:0 +LH:0 +end_of_record +TN: +SF:utils/credentials.py +DA:1,1,7Z9Lj4ed27Wf2hBX6jooEA +DA:3,1,oBn3m+aX07XSXyLZ7hhJvQ +DA:6,1,gvBB+eqYJmZEE370I4PLJw +DA:20,1,uMN5/uIZZnp9nEAOplwhrg +DA:22,1,YJqLCDyWmMT4Vt80XKej7A +DA:24,1,IxExvRJQkW08pwLjPeV+AA +DA:25,1,saUrv7XldTgMIzkqBUF3sw +DA:26,1,n5Fd4k6CRQXcqYOJ1AroeA +DA:27,1,5aSjAewLt9BgZu1M2CyLDg +DA:29,1,UrYRi5Ku4cGH9QytPbwuIg +DA:32,1,Cm8bLUg146j1KAg+xEigfA +DA:46,1,uMN5/uIZZnp9nEAOplwhrg +DA:48,1,r0VtMIE142HhS1QHUy84/Q +DA:50,1,sJYVqUX9JungmATURLwWTw +DA:51,1,7kGLhyj2Gz1FgT9D0bb9Vg +DA:52,1,+GdQ45oCl4mUsGv9HojWfg +DA:53,1,e710G8kzqt5PjHHCzkMTng +DA:55,1,H17V4FJl1OEtok8qC+q/+A +DA:58,1,kSnEDTyHvsGabszAxYqSww +DA:72,1,uMN5/uIZZnp9nEAOplwhrg +DA:74,1,/vIX4NRiJ6gfFdtd7v/5CA +DA:76,1,LnlORR7UjQKf3WTSqC+LpQ +DA:77,1,l8Cclrt4B4dMSzJot1zPcg +DA:78,1,hIKLCkB8Re7lOm0QLJbg6A +DA:79,1,y1aZfDdlZokMd4D/4FAmwA +DA:81,1,T7zGOSr3oGWrSM/CzDYDcw +LF:26 +LH:26 +end_of_record +TN: +SF:utils/data_source_selector.py +DA:1,1,OXA5GZhWrVnCdJ6JVjaTww +DA:2,1,6XxZigYIXTEoXqTlBNKh4A +DA:5,1,cq+6nGZ4L624r1Qu2pX9IA +DA:6,1,bDTSD3NwJzYLX9VLOWNrog +DA:21,1,OfPQ5nOrizIBxrqsb+18Ww +DA:22,1,khL+YRyaqSGXvXfJuXRwfw +DA:23,1,9FG7XLm0rLCGHwL0YJNB/w +DA:24,1,j1mUH+lpTuJQfouULXJ7Kg +DA:26,1,fEBuiKF4P+NVarquP1gmjQ +DA:27,1,QFftjygbzu3/aoWrwPWGhQ +DA:28,1,tNHEj4JtLDnMXIsuX0QWgA +DA:37,1,1GeTxQg5T4sfLOBh5IROaw +DA:38,1,HVyrBJ61ni9ux6MdUIeaUA +DA:41,1,pK1ZUnMvS6wV8z2nLw6pKg +DA:43,1,wzxE+BQWKgUayLEFKTCYmQ +LF:15 +LH:15 +end_of_record +TN: +SF:utils/fetch_community_id.py +DA:1,1,cJoQvc8d9CEP56sPh71adA +DA:4,1,lmwdGMkCHlWXv2nNm/Uxdw +DA:19,1,YjVJLe7hbK7XTWsOoFVj/w +DA:21,1,QA9dGGPTeIp07ShApLAF4A +DA:24,1,U4Sjk6MLNWi4lp3eZMYMcg +DA:25,1,A4crhuYlU3KiAJlkPHK9QQ +DA:26,1,nqKcow6NrIpARXr+QJdOSg +DA:27,1,QKuzwW865EXj8VNYNgSC9w +LF:8 +LH:8 +end_of_record +TN: +SF:utils/mongo.py +DA:1,1,EZEzNmL93PY96zsSuI0PfQ +DA:2,1,gGTQvjsCFZRTAK01JA8k5Q +DA:4,1,aQa914DmkZxIUR93SEZpuw +DA:6,1,7wOattRMH258CtQs+UrpsQ +DA:9,1,yl+B0YFGI3t/m+5Suhwqfg +DA:10,1,83sxBcOsVIr2WiCqCQbLZQ +DA:12,1,gFdnwIzxJ64X+nD2UauP7A +DA:13,1,nedGLtP1l6hUktBMlnMhYA +DA:16,1,9pf1HZayXGTtSA9Rvl+WMg +DA:17,1,quD3RtvYZgmdln2jYSS7tA +DA:18,1,CUeVzuv4ZHDgYpULPWrh+A +DA:19,1,p8beMpKHz8de2xpXvL/QrQ +DA:21,1,KxgaGtJPkScc4OG7mJ59vg +DA:22,1,UE5ijQ2kRfIL7H25RlfGsg +DA:23,1,/k6qkyUHhH0yi+zpCWRQEg +DA:24,1,atc/IsM8O08RY1DyTj03hA +DA:25,1,xvWIcpcyNr0gyGcTkdMxXw +DA:26,1,wqqc+RCz9O1/BPQQ/avpsw +DA:27,1,pqGPD3HqI8BK+o/OexZo/A +DA:31,1,kixoR3OQ3VPgFNcOZnJMCw +DA:33,1,BN/xY+w6ECzfgEoHXowADw +DA:34,1,fgy3JEj8zwF+XOMFGgrcLQ +DA:37,1,ta6BcHtLzbVui7kSYavGHg +DA:38,1,hsfmwwQPyB4q/pQbrCvugg +DA:39,1,XDsjAHuILiKaPifDcNRk/A +DA:40,1,OLNS5e2ODcauq2phB8fr4A +DA:41,1,D4zo9qN1NT7YrDJBIRiK+A +DA:43,1,IBgmfx3PCWZ6RwfdTKp+Fg +DA:45,1,E2z+ad+nCvPlAX1NpKcGVw +DA:14,0,+QJTRb5rAKExUUr0WnnqAg +DA:28,0,LFZOE01yWzhpu4V3LUU0kw +DA:29,0,KQd83bxFJN5wqRs65ZHiAg +LF:32 +LH:29 +end_of_record +TN: +SF:utils/qdrant_utils.py +DA:1,1,ntJL14mrBgAly/E3tslOUg +DA:4,1,t9LFvlhmTNDcbcR06swJFA +DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ +DA:15,1,bxcqzIMUQ67BtZZ0H64HWQ +DA:16,1,lwSaYcYgqD6duWfx8iDyCg +DA:18,1,o9EuHdH+gpLJ8BZ0vpjMdg +DA:32,1,RI9Ka1LT1e/iffNnfq+bLg +DA:33,1,flM9IHJmqBjqVddLMZTPMg +DA:34,1,nAYiUkgpfqrBg3mJ07kzXA +LF:9 +LH:9 +end_of_record +TN: +SF:utils/query_engine/__init__.py +DA:2,1,Vnj8yLJpJ/wVehtFqro6mg +DA:3,1,Yen9HUSDB0Wd67+ldfKlng +DA:4,1,/+xElf2yXN2V7g/cx46R4w +DA:5,1,+faw0bsltOVhGJ4yjZDL9Q +DA:6,1,d8GL/JuynocTO/XRp6VV+Q +DA:7,1,J4oH3WJqtVVhhLElWxqk+A +LF:6 +LH:6 +end_of_record +TN: +SF:utils/query_engine/base_pg_engine.py +DA:1,1,VRsPke+iz1KOD5Ylfx/vow +DA:2,1,tv1rZhcNs3rWaZFnKxX8AA +DA:3,1,9zqOVuK2+mKhEfPQ8X750Q +DA:4,1,i4oCYN8ZAW8fFZiqSKryuQ +DA:5,1,1rLMUIEu7V96D7PKPUCUeQ +DA:6,1,wkfOwWcqhgnKREP2SoBl+A +DA:7,1,r/vTV1phZQGHkBIJdSWs9w +DA:10,1,1bcURF390sME4Zf7IEeg+A +DA:11,1,H8dNfXkovyjk0TB6ywYU0w +DA:25,1,NcYJVT1t86PY5bH1L4WMag +DA:26,1,lwSaYcYgqD6duWfx8iDyCg +DA:27,1,8CZJFonIL+691rDOKQSd8g +DA:29,1,LYmpQR52gMFxj5cQAgvBmw +DA:44,1,xOFEHQFRGWy9u/WDYjQNqw +DA:62,1,bAiir3mXHBEtXikoRFEOPw +DA:63,1,g/SCiHtEQsEcR+wXlI7gqQ +DA:66,1,C6ZtZYzya5SESpmY0jNCRw +DA:67,1,/NgtN11YZmhPFCQOzPd8bA +DA:69,1,DougZ9SUAHum4wQgjTyO4w +DA:71,1,xJlRV8Kxyu7FST2sQDK4bg +DA:73,1,v6Z2he5grryd4Z0ffKpgOA +DA:79,1,oaD/kc+MzLZ0/CzfGaJX2A +DA:80,1,ItkzfojqvwK4mDljHSDeZQ +DA:30,0,s4JEvRyP+y43QrQVUGI17A +DA:33,0,kihnT3XHavhUNbNr4ho+6g +DA:35,0,VBOpcm7qtCU0WDTcl8sXBw +DA:38,0,iXXJG8Mlu5JagixPsayHsw +DA:42,0,rRmN9Dhy/+TqccGt2h3m4A +LF:28 +LH:23 +end_of_record +TN: +SF:utils/query_engine/base_qdrant_engine.py +DA:1,1,tv1rZhcNs3rWaZFnKxX8AA +DA:2,1,9zqOVuK2+mKhEfPQ8X750Q +DA:3,1,Epduv66JxgjoxUaCT3nS2g +DA:6,1,1rLMUIEu7V96D7PKPUCUeQ +DA:7,1,6yg1SvuqaHW9gVNVzNYVrg +DA:10,1,HYj/N35l9bZAx7FsPcM3Zg +DA:11,1,H8dNfXkovyjk0TB6ywYU0w +DA:25,1,NcYJVT1t86PY5bH1L4WMag +DA:26,1,lwSaYcYgqD6duWfx8iDyCg +DA:27,1,nHZNxusJFY2cf/fBKlEIRw +DA:29,1,LYmpQR52gMFxj5cQAgvBmw +DA:30,1,3x3ebQUpyH0B5GozBlXRsg +DA:33,1,kihnT3XHavhUNbNr4ho+6g +DA:35,1,y/eQGOxxhvfE76tIHc6cwQ +DA:39,1,iXXJG8Mlu5JagixPsayHsw +DA:43,1,rRmN9Dhy/+TqccGt2h3m4A +DA:45,1,xOFEHQFRGWy9u/WDYjQNqw +DA:61,1,cNSMxn65O/ZzNyMdX/7n0g +DA:62,1,o0upLLxBXoxpEIXpEFDjrw +DA:66,1,ISnP9S4BsPcPJGVvKfraWA +DA:67,1,ItkzfojqvwK4mDljHSDeZQ +LF:21 +LH:21 +end_of_record +TN: +SF:utils/query_engine/gdrive.py +DA:1,1,C8DhKIOrNkqmLLr7blG7vw +DA:4,1,66+KyTBbTDXPKLFpM19sEg +DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ +DA:6,1,/Hs5KWsUByIj7S7tOxnUAw +DA:7,1,QC64UFI8neRNXWDeZEm1Zg +LF:5 +LH:5 +end_of_record +TN: +SF:utils/query_engine/github.py +DA:1,1,C8DhKIOrNkqmLLr7blG7vw +DA:4,1,ZADGWypDUyWGLHxGn9Lzog +DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ +DA:6,1,+B610GO5jHoGxR6V4GkacA +DA:7,1,QC64UFI8neRNXWDeZEm1Zg +LF:5 +LH:5 +end_of_record +TN: +SF:utils/query_engine/level_based_platform_query_engine.py +DA:1,1,EZEzNmL93PY96zsSuI0PfQ +DA:3,1,nOgG6xCCb4HBDSN3x9QS5A +DA:4,1,xeVnH2DNgPEjUcwgJVQehQ +DA:5,1,tv1rZhcNs3rWaZFnKxX8AA +DA:6,1,3H1a4ALTm6+CsKOSplE0HQ +DA:7,1,DDgKR+7enopNKdkqI8I62w +DA:8,1,22ORhkU2P3imK0vCquiKNQ +DA:9,1,WKJgOHAAbKxKt/3sZtK63g +DA:13,1,7FwFHH3k33riE6ReKUYWUw +DA:14,1,ZwFry2q3ybNGgdFz5V3nNw +DA:15,1,DCDK4Yl+/ELe6m+70OYJ1g +DA:16,1,gIY0rCnKSLWVSRd76jn1Mg +DA:17,1,sJebj/U8hMCZj5D1eM0aJg +DA:19,1,YjzuKhn/xRwHSBjOcQhsdA +DA:31,1,aAaxbH5EhJE64W9zBhIF9g +DA:32,1,3Yt4nvHJdyo2mqZ49Tnj5w +DA:33,1,wtC4abIbxsngso+ns9k6vw +DA:34,1,kAXWcSibUmRxDnaE5g0mJQ +DA:35,1,lyWcrb86KF2Hcx929BzmxQ +DA:37,1,p2Qx/bziJstmcY6BKOsgBA +DA:56,1,Nnqu9X+lU2lagW39QNhFWA +DA:57,1,aQZXZHzGnOZY9rrOd6/wvQ +DA:112,1,+AoWudgyI70lgbtxX7lqig +DA:114,1,5YOAMYmqtQNtDb2irXQR0w +DA:117,1,y55N4o1rvRJyLK/0r/5L7A +DA:118,1,WXOqRntq6K1mGRBen+UcbA +DA:119,1,zBvKWW8jFIhbdTFZsnTXVA +DA:120,1,lLRy3eVWI8Pt8RNaSKBsug +DA:126,1,TC5ZkNxN5KKk7z8DTijqjg +DA:128,1,O8d2LBsxkTL8X4P7tH1g7A +DA:129,1,r9RgqFbQ/+jUw8oOkCWalw +DA:138,1,mkH3WvSUvL2/3P24Hu7cKA +DA:139,1,h4T9Vvca+5tmqAtyyyXNyg +DA:141,1,tDdqYg3dq/wD/tVD8vu9/g +DA:143,1,KTtT/NxaurUIwrbxcvtcaA +DA:144,1,yKiHkJzAUkqGbTUr32ub9Q +DA:145,1,kj6Z4kNQl+AXmKR+j3zAYA +DA:147,1,nuShR7aDtL9Tmh7gKu9FIw +DA:154,1,Nnqu9X+lU2lagW39QNhFWA +DA:155,1,lafnpJ+CQMAdI8iasAI41A +DA:198,1,+AoWudgyI70lgbtxX7lqig +DA:199,1,c6q+PcdvoDVhU0DNZ1PvZg +DA:201,1,8rFzc+DLQVEr0zGZDWPbCw +DA:206,1,IYUyGR2ggWDMKtKUakHiKw +DA:209,1,7JZtaXtmzqywgswRvQt7+A +DA:211,1,jepe1mImdRVnsx4Js59XKg +DA:216,1,58c+JAyIdGn+tNpgL+tKQg +DA:219,1,J19yyj1rBA+MjKawC6oHOg +DA:223,1,V/hzcQrmu7D3FUS+Uvd8tA +DA:229,1,kn8wpgTeUaT9YyomopUMhA +DA:239,1,evg8qzxYp7Is+U7oBQ4HWg +DA:240,1,izA1BjWd1igKsXjjoKSKQw +DA:244,1,C6bghf8IPyZTFyYwr+agEQ +DA:245,1,9ztu/13sn+eyjgOcCfcsGQ +DA:246,1,J6uEf5KSYdEBC2a35vgBfw +DA:247,1,zRB9nprox4hkMNnasm1DfQ +DA:248,1,h4T9Vvca+5tmqAtyyyXNyg +DA:249,1,0fJTQHj5cDj1/stYiH+PTw +DA:251,1,HJIEUG2cMkYCGEwwS43VRw +DA:255,1,PH3ft6eUMPDGM/KLBdxVvA +DA:262,1,sgXKOyybp4yEVrJLhkOkkQ +DA:264,1,SHylApe6S33xsDgn0BMOCA +DA:270,1,hRgQD5up9E4XZrIkKN425g +DA:272,1,U9dLMFnDTp4r1/r38zkvng +DA:273,1,VdJ5KSGdN1AmU/IO2DW8vg +DA:277,1,pu4uDADWR92AgQ4lElnXng +DA:280,1,C87Lpztr4KKJLFNDJA1OTQ +DA:308,1,9/pmpwUZHacFljsBY7h3sA +DA:309,1,at/HVooxpizTsAJ4uAUG7w +DA:314,1,KXVUWpE1ypka5ImuW1QMpA +DA:320,1,VH4nv2g4YIj+iGTWpSQYIA +DA:323,1,LrdimJrx2agAR/ZKdZ1A9A +DA:344,1,kXJz5rsWvT06VgqAoVleRw +DA:346,1,uBcLKYMt4YuMysNBii+5yw +DA:40,0,jepe1mImdRVnsx4Js59XKg +DA:45,0,mHs0z5vDWZxuReGU6yh4VQ +DA:49,0,9+0wI5fq5LfHNN4soQijpg +DA:50,0,GkNF20k0ddXAQjklAJZQmA +DA:51,0,b9ewJXUvjgWBkQaDeA8l8w +DA:52,0,ty2ljlDXsDfuBiNGV+Op0g +DA:54,0,KPoxs3J/EBRSZ4FRpyqT9Q +DA:242,0,9FqK6DjAyzNXTmh2W+ebfw +DA:281,0,FjBQWq9IG4J6Cm7RfALnWA +DA:286,0,Gz5HHhWWOcMzaSURTUFBUw +DA:294,0,at/HVooxpizTsAJ4uAUG7w +DA:297,0,9/pmpwUZHacFljsBY7h3sA +DA:298,0,KXVUWpE1ypka5ImuW1QMpA +DA:304,0,VH4nv2g4YIj+iGTWpSQYIA +DA:324,0,6ChizqLUbucwiNDDkJLrDA +DA:328,0,5lqBTTJ5h43YzUS8pQ4tog +DA:333,0,eZP1y6iLmUAlnhAelnCupw +DA:336,0,5YBDint/C2L/36Y30bS/pg +DA:342,0,lMU2/UJ7v9WOAuw2dFSC7A +LF:93 +LH:74 +end_of_record +TN: +SF:utils/query_engine/level_based_platforms_util.py +DA:1,1,EZEzNmL93PY96zsSuI0PfQ +DA:3,1,KA9tLtxJCmXGWOalg5c/zA +DA:4,1,ZwFry2q3ybNGgdFz5V3nNw +DA:7,1,1n0+DB+0GKrFV9FTWNebSA +DA:8,1,2rHnDUvBBQWvCCEwdSz4mw +DA:9,1,PeQpl/2nVKsQpLWJOzEqQw +DA:10,1,zRgFE7FqhvZdMz1X95SX7Q +DA:11,1,GHz25t/A39Sr/cnYnniNCQ +DA:13,1,HNZxq1ze/z5tqRDtNZKTdA +DA:20,1,UBWWkmgHPHVfsMCI1Fsajw +DA:38,1,uBcLKYMt4YuMysNBii+5yw +DA:40,1,3z0QwSAFw5zRa9xwH+rgEA +DA:62,1,eLOsbqmnKW0+8kNbMCMDLg +DA:65,1,osmxeRTnhC30Fq8zkebgGw +DA:67,1,qZMvO0jjLdJVah0rd8S/jw +DA:68,1,Qkf8eNyjQqG0IH7RqJQ3+w +DA:69,1,PY15Nh0HkYgRKH6u93BSJQ +DA:70,1,68Yt75DazluknIc6D2DJXw +DA:73,1,GmbNDixOUUQ8VA997/7nbw +DA:77,1,mItPeXzAYUKXS9vXTJTWYw +DA:79,1,IsCQ9tgCFPPgq6WnIOsimw +DA:81,1,DvnMWQ7HgHURrxiyFIgH4g +DA:99,1,hRgQD5up9E4XZrIkKN425g +DA:101,1,9dqwsZzGUS6Yl4TftdS6GQ +DA:103,1,gJ439ucMTM4q5oT2Tov/Lw +DA:107,1,e440eUgPColUOy6pY34qVQ +DA:108,1,2SGzQDtqdvV4c37zEjo8Mw +DA:109,1,YSXhRM4A5II45l7DOsPJzw +DA:110,1,fPZKnSTL7HTWx6ZDYv5gLA +DA:113,1,rf+yiaZsFd2skz8OTryeUA +DA:118,1,sRbiZiQQFviDtVPK4JvfLg +DA:119,1,n6gf1L4xPtondA2Fo95wQg +DA:125,1,jqi+MeL/wMiw3ev5TkYnbw +DA:127,1,zJ63EizQBDp4hcpXusKZkw +DA:128,1,oNfSos/b+wErmWh2QP+c8A +DA:154,1,JnuqTw09b0rXs7pyJ5g2sQ +DA:158,1,q8024D2AVZGkIsSbhHUOdQ +DA:159,1,pLfMADHw+eGw8ECl0Z5aSw +DA:160,1,E8AYJN10pVbhOioFIMeHAA +DA:167,1,RczncNMSMdyFin4BxazGXg +DA:176,1,XTmIht7CQjYJLiUKNyewJQ +DA:179,1,kIUZ7J+O00xTJkiiIyNlaA +DA:183,1,UVbveT4931BLF02gVy8O/w +DA:184,1,noAL/2Aq9wHOV6+GxdUZzw +DA:190,1,F4xCoijosIqQAkElo3fWEg +DA:135,0,gFvTFqI1qeC4aC0n/wkg7g +DA:138,0,oNfSos/b+wErmWh2QP+c8A +DA:145,0,Y+9Cpj0wdzCrYKkYoQEr2w +DA:147,0,oNfSos/b+wErmWh2QP+c8A +LF:49 +LH:45 +end_of_record +TN: +SF:utils/query_engine/media_wiki.py +DA:1,1,C8DhKIOrNkqmLLr7blG7vw +DA:4,1,NRapDFy+V5xvFfDLz5AwKw +DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ +DA:6,1,0DvXG5U+Mac124OTxMZ50g +DA:7,1,QC64UFI8neRNXWDeZEm1Zg +LF:5 +LH:5 +end_of_record +TN: +SF:utils/query_engine/notion.py +DA:1,1,C8DhKIOrNkqmLLr7blG7vw +DA:4,1,taUtbU69r/u+KwuhqbZXwQ +DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ +DA:6,1,WApUn0uoi5HwYH8FxulgWw +DA:7,1,QC64UFI8neRNXWDeZEm1Zg +LF:5 +LH:5 +end_of_record +TN: +SF:utils/query_engine/prepare_discord_query_engine.py +DA:1,1,FI1BbsECoGeHqt8kW925Kg +DA:3,1,O/Qn0s4Cmfehw44CStHXEw +DA:6,1,hvsmyd45Rodf/tBfNPWgPw +DA:34,1,q+12rfq5jrwpiWDQodqLhg +DA:35,1,eQava1okCaI1vmj/z0O7vw +DA:41,1,v4mNTOz+svYfaDNdHA++JA +DA:44,1,4OTfIyNPew3rgYQKc0Uofw +DA:68,0,oXzDsL38DFI/SF5Wkrqijg +DA:77,0,YRW/xAcZ0a28FDbybpQOAQ +LF:9 +LH:7 +end_of_record +TN: +SF:utils/query_engine/prepare_discourse_query_engine.py +DA:1,1,FI1BbsECoGeHqt8kW925Kg +DA:3,1,O/Qn0s4Cmfehw44CStHXEw +DA:6,1,SJPP3rgXO4WKqYx8e9xnJA +DA:31,1,q+12rfq5jrwpiWDQodqLhg +DA:32,1,eQava1okCaI1vmj/z0O7vw +DA:39,1,v4mNTOz+svYfaDNdHA++JA +DA:42,1,ts/D49naBdnJYvavhOMruw +DA:65,0,oXzDsL38DFI/SF5Wkrqijg +DA:73,0,YRW/xAcZ0a28FDbybpQOAQ +LF:9 +LH:7 +end_of_record +TN: +SF:utils/query_engine/subquery_gen_prompt.py +DA:1,1,OaTlemF3bXMi5MP7Db8vkQ +DA:2,1,da4F/HJkmdp35fCZoBYhug +DA:4,1,0+n0VDq2WJj6tgatFS0N1A +DA:5,1,p7jXd7VVorunNziWj6uTSQ +DA:6,1,3qjx/WnOrCQjYiF4JZTaTw +DA:9,1,JbdHvfcPRGbdQrtxirTU3Q +DA:10,1,GO/ZdycP15VWEePnEr38TQ +DA:11,1,O0J31uRmy1IE8NTEkT6wjA +DA:12,1,xUsGrHzeZSrjujDYvG8rqQ +DA:13,1,yqdV1UNZupsu1vbhHyQTNw +DA:16,1,YShAQ9o49XB7DbWwXU+SiA +DA:23,1,olsSoI2ZhROpde/oLnsWsA +DA:27,1,hgSlxMu28mzot0TyCC/FXA +DA:37,1,oH4MclLKdozptQeZO7CXFA +DA:38,1,iNdFOle/XUxgJDNuyCqOpQ +DA:46,1,PKlGrs30MCJpf45GqcuU3w +DA:48,1,FOJMonFW9MigPvGx3lVkRA +DA:70,1,d5ujy6JQKRcuZhFUBFDqMA +DA:83,1,K/JLImFIZiBQYyJ5dS7jTQ +DA:84,1,rU3AJeD7nrlk8aGyggdiXQ +LF:20 +LH:20 +end_of_record diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 0eccf2e..b25e239 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -5,10 +5,9 @@ services: build: context: . target: prod - dockerfile: Dockerfile + command: python3 server.py worker: build: context: . target: prod dockerfile: Dockerfile - command: python3 worker.py diff --git a/worker.py b/server.py similarity index 97% rename from worker.py rename to server.py index 8224baf..91d0dae 100644 --- a/worker.py +++ b/server.py @@ -3,7 +3,7 @@ from typing import Any import backoff -from celery_app.tasks import ask_question_auto_search +from worker.tasks import ask_question_auto_search from pika.exceptions import ConnectionClosedByBroker from tc_messageBroker import RabbitMQ from tc_messageBroker.rabbit_mq.event import Event diff --git a/celery_app/__init__.py b/worker/__init__.py similarity index 100% rename from celery_app/__init__.py rename to worker/__init__.py diff --git a/celery_app/tasks.py b/worker/tasks.py similarity index 98% rename from celery_app/tasks.py rename to worker/tasks.py index 8667231..ca91d5c 100644 --- a/celery_app/tasks.py +++ b/worker/tasks.py @@ -5,8 +5,8 @@ from typing import Any from celery.signals import task_postrun -from celery_app.server import app -from celery_app.utils.fire_event import job_send +from worker.worker import app +from worker.utils.fire_event import job_send from dotenv import load_dotenv from subquery import query_multiple_source from tc_messageBroker.rabbit_mq.event import Event diff --git a/celery_app/utils/__init__.py b/worker/utils/__init__.py similarity index 100% rename from celery_app/utils/__init__.py rename to worker/utils/__init__.py diff --git a/celery_app/utils/fire_event.py b/worker/utils/fire_event.py similarity index 100% rename from celery_app/utils/fire_event.py rename to worker/utils/fire_event.py diff --git a/celery_app/server.py b/worker/worker.py similarity index 57% rename from celery_app/server.py rename to worker/worker.py index 499aff0..3a3a0d5 100644 --- a/celery_app/server.py +++ b/worker/worker.py @@ -7,5 +7,8 @@ host = rabbit_creds["host"] port = rabbit_creds["port"] -app = Celery("celery_app/tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//") -app.autodiscover_tasks(["celery_app"]) +app = Celery("tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"]) +# app.autodiscover_tasks(["celery_app"]) + +if __name__ == '__main__': + app.start() \ No newline at end of file From 204ad4fab2900ee85e2c0d32e01629d6eda3e2ed Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 09:58:54 +0200 Subject: [PATCH 02/50] missing document start "---" (document-start) --- .github/workflows/production.yml | 2 ++ .github/workflows/start.staging.yml | 2 ++ .gitignore | 1 + docker-compose.example.yml | 2 +- docker-compose.test.yml | 17 +++++++++++------ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 940fb0e..39d3f14 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -1,3 +1,5 @@ +--- + name: Production CI/CD Pipeline on: diff --git a/.github/workflows/start.staging.yml b/.github/workflows/start.staging.yml index 24e53da..a53de6e 100644 --- a/.github/workflows/start.staging.yml +++ b/.github/workflows/start.staging.yml @@ -1,3 +1,5 @@ +--- + name: Staging CI/CD Pipeline on: pull_request diff --git a/.gitignore b/.gitignore index 32ea46e..36af5d5 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml .hypothesis/ .pytest_cache/ cover/ +coverage/ # Translations *.mo diff --git a/docker-compose.example.yml b/docker-compose.example.yml index b25e239..aa671f8 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -1,4 +1,4 @@ -version: "3.9" +--- services: server: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index e1a1fb7..9325472 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,4 +1,4 @@ -version: "3.9" +--- services: app: @@ -55,7 +55,7 @@ services: - NEO4J_PLUGINS=["apoc", "graph-data-science"] - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.* healthcheck: - test: ["CMD", "wget", "http://localhost:7474"] + test: [ "CMD", "wget", "http://localhost:7474" ] interval: 1m30s timeout: 10s retries: 2 @@ -77,7 +77,7 @@ services: - POSTGRES_USER=root - POSTGRES_PASSWORD=pass healthcheck: - test: ["CMD-SHELL", "pg_isready"] + test: [ "CMD-SHELL", "pg_isready" ] interval: 10s timeout: 5s retries: 5 @@ -105,12 +105,17 @@ services: qdrant-healthcheck: restart: always image: curlimages/curl:latest - entrypoint: ["/bin/sh", "-c", "--", "while true; do sleep 30; done;"] + entrypoint: + [ + "/bin/sh", + "-c", + "--", + "while true; do sleep 30; done;" + ] depends_on: - qdrant healthcheck: - test: ["CMD", "curl", "-f", "http://qdrant:6333/readyz"] + test: [ "CMD", "curl", "-f", "http://qdrant:6333/readyz" ] interval: 10s timeout: 2s retries: 5 - From e100dd201af4c8d21b8b1cded5e533ae11afe9a6 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:09:15 +0200 Subject: [PATCH 03/50] add .yamllint --- .yamllint | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..dd74efc --- /dev/null +++ b/.yamllint @@ -0,0 +1,7 @@ +rules: + brackets: + forbid: false + min-spaces-inside: 1 + max-spaces-inside: 1 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 \ No newline at end of file From f8900473fc251414d8fc60e37edf59ae0d4f9ed1 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:13:37 +0200 Subject: [PATCH 04/50] fix imports (isort) --- server.py | 8 ++++---- worker/tasks.py | 19 ++++++++++--------- worker/worker.py | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/server.py b/server.py index 91d0dae..6fa711b 100644 --- a/server.py +++ b/server.py @@ -3,16 +3,16 @@ from typing import Any import backoff -from worker.tasks import ask_question_auto_search from pika.exceptions import ConnectionClosedByBroker from tc_messageBroker import RabbitMQ from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.queue import Queue + from utils.credentials import load_rabbitmq_credentials from utils.fetch_community_id import fetch_community_id_by_guild_id +from worker.tasks import ask_question_auto_search def query_llm(recieved_data: dict[str, Any]): diff --git a/worker/tasks.py b/worker/tasks.py index ca91d5c..6d3d917 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -5,21 +5,20 @@ from typing import Any from celery.signals import task_postrun -from worker.worker import app -from worker.utils.fire_event import job_send from dotenv import load_dotenv -from subquery import query_multiple_source from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( - InteractionCallbackData, -) -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ + InteractionCallbackData +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue from traceloop.sdk import Traceloop + +from subquery import query_multiple_source from utils.data_source_selector import DataSourceSelector +from worker.utils.fire_event import job_send +from worker.worker import app @app.task @@ -109,6 +108,7 @@ def ask_question_auto_search( queue_name=Queue.DISCORD_BOT, content=response_payload, ) + logging.info("FINISHED JOB") except Exception as exp: logging.error(f"Exception {exp} | during processing the question {question}") response_payload = Payload.DISCORD_BOT.INTERACTION_RESPONSE.Edit( @@ -122,6 +122,7 @@ def ask_question_auto_search( queue_name=Queue.DISCORD_BOT, content=response_payload, ) + logging.info("FINISHED JOB WITH EXCEPTION") @task_postrun.connect diff --git a/worker/worker.py b/worker/worker.py index 3a3a0d5..a12f74b 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -1,4 +1,5 @@ from celery import Celery + from utils.credentials import load_rabbitmq_credentials rabbit_creds = load_rabbitmq_credentials() From 37e4733599a214070dea6d2d7d627acb6e3e44e9 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:24:28 +0200 Subject: [PATCH 05/50] black lint --- server.py | 5 +++-- worker/tasks.py | 10 ++++++---- worker/worker.py | 10 +++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/server.py b/server.py index 6fa711b..778d3a1 100644 --- a/server.py +++ b/server.py @@ -6,8 +6,9 @@ from pika.exceptions import ConnectionClosedByBroker from tc_messageBroker import RabbitMQ from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.queue import Queue from utils.credentials import load_rabbitmq_credentials diff --git a/worker/tasks.py b/worker/tasks.py index 6d3d917..1bf3e65 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -7,10 +7,12 @@ from celery.signals import task_postrun from dotenv import load_dotenv from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ - InteractionCallbackData -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( + InteractionCallbackData, +) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue from traceloop.sdk import Traceloop diff --git a/worker/worker.py b/worker/worker.py index a12f74b..3e1bb1e 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -8,8 +8,12 @@ host = rabbit_creds["host"] port = rabbit_creds["port"] -app = Celery("tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"]) +app = Celery( + "tasks", + broker=f"pyamqp://{user}:{password}@{host}:{port}//", + include=["worker.tasks"], +) # app.autodiscover_tasks(["celery_app"]) -if __name__ == '__main__': - app.start() \ No newline at end of file +if __name__ == "__main__": + app.start() From 2630edbbfda3c8be6d320e15b267d4e56a09ff1d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:29:33 +0200 Subject: [PATCH 06/50] isort fix --- server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index 778d3a1..6fa711b 100644 --- a/server.py +++ b/server.py @@ -6,9 +6,8 @@ from pika.exceptions import ConnectionClosedByBroker from tc_messageBroker import RabbitMQ from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.queue import Queue from utils.credentials import load_rabbitmq_credentials From 9d48fcc7d24db59fdcdacb45f21f6ea6b1da2c74 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:37:34 +0200 Subject: [PATCH 07/50] isort tasks --- worker/tasks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 1bf3e65..6d3d917 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -7,12 +7,10 @@ from celery.signals import task_postrun from dotenv import load_dotenv from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( - InteractionCallbackData, -) -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ + InteractionCallbackData +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue from traceloop.sdk import Traceloop From 8bcecb9eee49e0bc418502c341dd76ab75414cf1 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:42:23 +0200 Subject: [PATCH 08/50] black server --- server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 6fa711b..778d3a1 100644 --- a/server.py +++ b/server.py @@ -6,8 +6,9 @@ from pika.exceptions import ConnectionClosedByBroker from tc_messageBroker import RabbitMQ from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.queue import Queue from utils.credentials import load_rabbitmq_credentials From cda1315bd5be3dcc9dc2b60dde6ce646eea5b11d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 10:48:41 +0200 Subject: [PATCH 09/50] disabled isort --- .isort.cfg | 2 ++ worker/tasks.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..b871833 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +check = false diff --git a/worker/tasks.py b/worker/tasks.py index 6d3d917..1bf3e65 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -7,10 +7,12 @@ from celery.signals import task_postrun from dotenv import load_dotenv from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ - InteractionCallbackData -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( + InteractionCallbackData, +) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue from traceloop.sdk import Traceloop From b3a90e5d5e4b4ce9ea3d73cb861de7b5efd347a0 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 11:24:31 +0200 Subject: [PATCH 10/50] fix worker --- worker/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/worker.py b/worker/worker.py index 3e1bb1e..28e0764 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -9,7 +9,7 @@ port = rabbit_creds["port"] app = Celery( - "tasks", + "worker", broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"], ) From 0a5ef7c666848aff03137b85cb5a91df61fb32b0 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 11:33:12 +0200 Subject: [PATCH 11/50] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fcb40af..00e5a00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ RUN chmod +x docker-entrypoint.sh CMD ["./docker-entrypoint.sh"] FROM base AS prod -CMD ["celery", "-A", "tasks", "worker", "-l", "INFO"] +CMD ["celery", "-A", "worker", "worker", "-l", "INFO"] From 9726f9d8903dfce016caa659a2c1cc5be723ab42 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 11:38:34 +0200 Subject: [PATCH 12/50] fix worker --- worker/{worker.py => celery.py} | 2 +- worker/tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename worker/{worker.py => celery.py} (96%) diff --git a/worker/worker.py b/worker/celery.py similarity index 96% rename from worker/worker.py rename to worker/celery.py index 28e0764..3e1bb1e 100644 --- a/worker/worker.py +++ b/worker/celery.py @@ -9,7 +9,7 @@ port = rabbit_creds["port"] app = Celery( - "worker", + "tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"], ) diff --git a/worker/tasks.py b/worker/tasks.py index 1bf3e65..cbdf0bc 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -20,7 +20,7 @@ from subquery import query_multiple_source from utils.data_source_selector import DataSourceSelector from worker.utils.fire_event import job_send -from worker.worker import app +from worker.celery import app @app.task From 77576addf19531837f788e1ad9dcbbca51837788 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 11:51:18 +0200 Subject: [PATCH 13/50] move traceloop to celery --- worker/celery.py | 6 ++++++ worker/tasks.py | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/worker/celery.py b/worker/celery.py index 3e1bb1e..150a088 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,4 +1,7 @@ from celery import Celery +from dotenv import load_dotenv +from traceloop.sdk import Traceloop +import os from utils.credentials import load_rabbitmq_credentials @@ -16,4 +19,7 @@ # app.autodiscover_tasks(["celery_app"]) if __name__ == "__main__": + load_dotenv() + otel_endpoint = os.getenv("TRACELOOP_BASE_URL") + Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) app.start() diff --git a/worker/tasks.py b/worker/tasks.py index cbdf0bc..9bf3b8f 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -1,11 +1,9 @@ import gc import json import logging -import os from typing import Any from celery.signals import task_postrun -from dotenv import load_dotenv from tc_messageBroker.rabbit_mq.event import Event from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( InteractionCallbackData, @@ -15,7 +13,6 @@ ) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue -from traceloop.sdk import Traceloop from subquery import query_multiple_source from utils.data_source_selector import DataSourceSelector @@ -47,9 +44,6 @@ def ask_question_auto_search( - `date` - `content`: which is the `ChatInputCommandInteraction` as a dictionary """ - load_dotenv() - otel_endpoint = os.getenv("TRACELOOP_BASE_URL") - Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) prefix = f"COMMUNITY_ID: {community_id} | " logging.info(f"{prefix}Processing question!") From c1f61c8fd5a98b8d47f9ddf0f1a10f380f7bc8a3 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 12:30:22 +0200 Subject: [PATCH 14/50] use worker_process_init --- worker/celery.py | 8 -------- worker/tasks.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/worker/celery.py b/worker/celery.py index 150a088..35373c5 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,8 +1,4 @@ from celery import Celery -from dotenv import load_dotenv -from traceloop.sdk import Traceloop -import os - from utils.credentials import load_rabbitmq_credentials rabbit_creds = load_rabbitmq_credentials() @@ -16,10 +12,6 @@ broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"], ) -# app.autodiscover_tasks(["celery_app"]) if __name__ == "__main__": - load_dotenv() - otel_endpoint = os.getenv("TRACELOOP_BASE_URL") - Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) app.start() diff --git a/worker/tasks.py b/worker/tasks.py index 9bf3b8f..c539305 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -3,7 +3,7 @@ import logging from typing import Any -from celery.signals import task_postrun +from celery.signals import task_postrun, worker_process_init from tc_messageBroker.rabbit_mq.event import Event from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( InteractionCallbackData, @@ -19,6 +19,16 @@ from worker.utils.fire_event import job_send from worker.celery import app +from dotenv import load_dotenv +from traceloop.sdk import Traceloop +import os + + +@worker_process_init.connect(weak=False) +def init_tracing(): + load_dotenv() + otel_endpoint = os.getenv("TRACELOOP_BASE_URL") + Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) @app.task def ask_question_auto_search( From 75e784ca8074a863257e7ea2c3bd294a3789d97b Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:04:41 +0200 Subject: [PATCH 15/50] black --- worker/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/worker/tasks.py b/worker/tasks.py index c539305..a77d6ee 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -30,6 +30,7 @@ def init_tracing(): otel_endpoint = os.getenv("TRACELOOP_BASE_URL") Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) + @app.task def ask_question_auto_search( question: str, From cfbf9f7d30e3a7a403bbbab31cfc9b40eb6bb2a8 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:13:20 +0200 Subject: [PATCH 16/50] log trace init --- worker/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/worker/tasks.py b/worker/tasks.py index a77d6ee..c8116a1 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -26,9 +26,11 @@ @worker_process_init.connect(weak=False) def init_tracing(): + logging.info("Initializing trace...") load_dotenv() otel_endpoint = os.getenv("TRACELOOP_BASE_URL") Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) + logging.info("Trace initialized...") @app.task From 7bb0979a05f67f773b68e0e0e512d583049ae21a Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:19:12 +0200 Subject: [PATCH 17/50] test traceloop --- worker/celery.py | 13 +++++++++++++ worker/tasks.py | 15 ++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/worker/celery.py b/worker/celery.py index 35373c5..98a7806 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,12 +1,25 @@ +import logging from celery import Celery from utils.credentials import load_rabbitmq_credentials +from dotenv import load_dotenv +from traceloop.sdk import Traceloop +import os + +load_dotenv() + rabbit_creds = load_rabbitmq_credentials() user = rabbit_creds["user"] password = rabbit_creds["password"] host = rabbit_creds["host"] port = rabbit_creds["port"] +def init_tracing(): + logging.info("Initializing trace...") + otel_endpoint = os.getenv("TRACELOOP_BASE_URL") + Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) + logging.info("Trace initialized...") + app = Celery( "tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", diff --git a/worker/tasks.py b/worker/tasks.py index c8116a1..5b8c5a2 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -19,20 +19,9 @@ from worker.utils.fire_event import job_send from worker.celery import app -from dotenv import load_dotenv -from traceloop.sdk import Traceloop -import os - - -@worker_process_init.connect(weak=False) -def init_tracing(): - logging.info("Initializing trace...") - load_dotenv() - otel_endpoint = os.getenv("TRACELOOP_BASE_URL") - Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) - logging.info("Trace initialized...") - +from traceloop.sdk.decorators import task +@task() @app.task def ask_question_auto_search( question: str, From d2fb66583b50be6ef187d90178087d4b2788575d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:19:31 +0200 Subject: [PATCH 18/50] formatting --- worker/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/worker/tasks.py b/worker/tasks.py index 5b8c5a2..833b4bb 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -21,6 +21,7 @@ from traceloop.sdk.decorators import task + @task() @app.task def ask_question_auto_search( From 397164f050247e8699eb53ab6b5e9a406bf92e48 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:23:42 +0200 Subject: [PATCH 19/50] reformat --- worker/celery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/worker/celery.py b/worker/celery.py index 98a7806..1292f15 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -14,12 +14,14 @@ host = rabbit_creds["host"] port = rabbit_creds["port"] + def init_tracing(): logging.info("Initializing trace...") otel_endpoint = os.getenv("TRACELOOP_BASE_URL") Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) logging.info("Trace initialized...") + app = Celery( "tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", From f206fad1b8a8917cedfda077fe03aef8b18328e4 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:42:21 +0200 Subject: [PATCH 20/50] fix decorator? --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index 833b4bb..eed0005 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,7 +22,7 @@ from traceloop.sdk.decorators import task -@task() +@task @app.task def ask_question_auto_search( question: str, From 375cd2316a53044a0b55f578ba65c6c76a20aa49 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:50:27 +0200 Subject: [PATCH 21/50] flip tasks --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index eed0005..b2dfb2f 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,8 +22,8 @@ from traceloop.sdk.decorators import task -@task @app.task +@task def ask_question_auto_search( question: str, community_id: str, From d857289513288dd2f0a7cc4694b957433caa2adc Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 13:56:46 +0200 Subject: [PATCH 22/50] remove @task --- worker/tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index b2dfb2f..6f0d6a7 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -19,11 +19,8 @@ from worker.utils.fire_event import job_send from worker.celery import app -from traceloop.sdk.decorators import task - @app.task -@task def ask_question_auto_search( question: str, community_id: str, From eae799925bb2305157a24fd159fe1b2f6e4e0145 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 14:01:59 +0200 Subject: [PATCH 23/50] init tracing --- worker/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/worker/celery.py b/worker/celery.py index 1292f15..b7fe1a0 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -29,4 +29,5 @@ def init_tracing(): ) if __name__ == "__main__": - app.start() + init_tracing() + app.start() \ No newline at end of file From e23dcadf71c49965c204dbcdc126b971a8eb160d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 14:06:08 +0200 Subject: [PATCH 24/50] fix format --- worker/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/celery.py b/worker/celery.py index b7fe1a0..3c3b277 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -30,4 +30,4 @@ def init_tracing(): if __name__ == "__main__": init_tracing() - app.start() \ No newline at end of file + app.start() From 6c2d75b84e3950ff79518450d5a50d5267a57c50 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 14:22:02 +0200 Subject: [PATCH 25/50] init tracing? --- worker/celery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/worker/celery.py b/worker/celery.py index 3c3b277..0938fb1 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -16,10 +16,10 @@ def init_tracing(): - logging.info("Initializing trace...") + logging.info("Initializing traceloop...") otel_endpoint = os.getenv("TRACELOOP_BASE_URL") Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) - logging.info("Trace initialized...") + logging.info("Traceloop initialized.") app = Celery( @@ -29,5 +29,5 @@ def init_tracing(): ) if __name__ == "__main__": - init_tracing() app.start() + init_tracing() From 3e43510905455b774c703b04ad36b3e55cc4d171 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 15:21:52 +0200 Subject: [PATCH 26/50] more logs --- worker/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/worker/celery.py b/worker/celery.py index 0938fb1..787f15a 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -22,6 +22,7 @@ def init_tracing(): logging.info("Traceloop initialized.") +init_tracing() app = Celery( "tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", @@ -30,4 +31,4 @@ def init_tracing(): if __name__ == "__main__": app.start() - init_tracing() + logging.info("App started.") From 6de25229c319b0bc7dd681fbc72b6e7d28440a52 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 15:35:41 +0200 Subject: [PATCH 27/50] move trace to utils --- utils/traceloop.py | 15 +++++++++++++++ worker/celery.py | 20 ++++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 utils/traceloop.py diff --git a/utils/traceloop.py b/utils/traceloop.py new file mode 100644 index 0000000..c12d431 --- /dev/null +++ b/utils/traceloop.py @@ -0,0 +1,15 @@ +from dotenv import load_dotenv +from traceloop.sdk import Traceloop +import os +import logging + +load_dotenv() + + +def init_tracing(): + logging.info("Initializing traceloop...") + print("Initializing traceloop...") + otel_endpoint = os.getenv("TRACELOOP_BASE_URL") + Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) + logging.info("Traceloop initialized.") + print("Traceloop initialized.") diff --git a/worker/celery.py b/worker/celery.py index 787f15a..4119c9f 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,12 +1,8 @@ import logging from celery import Celery from utils.credentials import load_rabbitmq_credentials +from utils.traceloop import init_tracing -from dotenv import load_dotenv -from traceloop.sdk import Traceloop -import os - -load_dotenv() rabbit_creds = load_rabbitmq_credentials() user = rabbit_creds["user"] @@ -14,21 +10,17 @@ host = rabbit_creds["host"] port = rabbit_creds["port"] - -def init_tracing(): - logging.info("Initializing traceloop...") - otel_endpoint = os.getenv("TRACELOOP_BASE_URL") - Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) - logging.info("Traceloop initialized.") - - -init_tracing() app = Celery( "tasks", broker=f"pyamqp://{user}:{password}@{host}:{port}//", include=["worker.tasks"], ) + if __name__ == "__main__": + logging.info("Starting app...") + print("Starting app...") + init_tracing() app.start() logging.info("App started.") + print("App started.") From d79e5539b98a3722c57eaf9df0da0b467abf483d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 15:48:07 +0200 Subject: [PATCH 28/50] use worker_init --- utils/traceloop.py | 2 -- worker/celery.py | 6 ------ worker/tasks.py | 5 ++++- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/utils/traceloop.py b/utils/traceloop.py index c12d431..9c088ba 100644 --- a/utils/traceloop.py +++ b/utils/traceloop.py @@ -8,8 +8,6 @@ def init_tracing(): logging.info("Initializing traceloop...") - print("Initializing traceloop...") otel_endpoint = os.getenv("TRACELOOP_BASE_URL") Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) logging.info("Traceloop initialized.") - print("Traceloop initialized.") diff --git a/worker/celery.py b/worker/celery.py index 4119c9f..10ffaea 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,4 +1,3 @@ -import logging from celery import Celery from utils.credentials import load_rabbitmq_credentials from utils.traceloop import init_tracing @@ -18,9 +17,4 @@ if __name__ == "__main__": - logging.info("Starting app...") - print("Starting app...") - init_tracing() app.start() - logging.info("App started.") - print("App started.") diff --git a/worker/tasks.py b/worker/tasks.py index 6f0d6a7..77c91f5 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -3,7 +3,7 @@ import logging from typing import Any -from celery.signals import task_postrun, worker_process_init +from celery.signals import task_postrun, worker_init from tc_messageBroker.rabbit_mq.event import Event from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( InteractionCallbackData, @@ -18,6 +18,9 @@ from utils.data_source_selector import DataSourceSelector from worker.utils.fire_event import job_send from worker.celery import app +from utils.traceloop import init_tracing + +worker_init.connect(init_tracing) @app.task From 769a75dd01e39cc60a3c2c9d35717258203ebfd8 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 15:59:22 +0200 Subject: [PATCH 29/50] fix worker init --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index 77c91f5..43f1ad7 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -20,7 +20,7 @@ from worker.celery import app from utils.traceloop import init_tracing -worker_init.connect(init_tracing) +@worker_init.connect(init_tracing) @app.task From 146fa3704cbd21be076d1dfb94e2c6b58a7f15c0 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 16:03:47 +0200 Subject: [PATCH 30/50] fix --- worker/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index 43f1ad7..39f19aa 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -20,7 +20,10 @@ from worker.celery import app from utils.traceloop import init_tracing -@worker_init.connect(init_tracing) + +@worker_init.connect +def init(): + init_tracing() @app.task From e47cc7fa92c50b5e8de172318768a6b042b69279 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 16:12:20 +0200 Subject: [PATCH 31/50] fix init args --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index 39f19aa..3c16732 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,7 +22,7 @@ @worker_init.connect -def init(): +def init(sender=None, **kwargs): init_tracing() From 0ecd9708d9e376ac499cd96f1326820f983d7967 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 16:26:42 +0200 Subject: [PATCH 32/50] add alot of signals --- worker/tasks.py | 57 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 3c16732..e0aa36e 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -3,7 +3,15 @@ import logging from typing import Any -from celery.signals import task_postrun, worker_init +from celery.signals import ( + task_postrun, + worker_init, + celeryd_init, + worker_process_init, + worker_ready, + celeryd_after_setup, + task_prerun, +) from tc_messageBroker.rabbit_mq.event import Event from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( InteractionCallbackData, @@ -21,11 +29,6 @@ from utils.traceloop import init_tracing -@worker_init.connect -def init(sender=None, **kwargs): - init_tracing() - - @app.task def ask_question_auto_search( question: str, @@ -127,6 +130,48 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") +@celeryd_init.connect +def celeryd_init(sender=None, **kwargs): + logging.info("celeryd_init start.") + init_tracing() + logging.info("celeryd_init end.") + + +@celeryd_after_setup.connect +def celeryd_after_setup(sender=None, **kwargs): + logging.info("celeryd_after_setup start.") + init_tracing() + logging.info("celeryd_after_setup end.") + + +@worker_init.connect +def worker_init(sender=None, **kwargs): + logging.info("worker_init start.") + init_tracing() + logging.info("worker_init end.") + + +@worker_process_init.connect +def worker_process_init(sender=None, **kwargs): + logging.info("worker_process_init start.") + init_tracing() + logging.info("worker_process_init end.") + + +@worker_ready.connect +def worker_ready(sender=None, **kwargs): + logging.info("worker_ready start.") + init_tracing() + logging.info("worker_ready end.") + + +@task_prerun.connect +def task_prerun(sender=None, **kwargs): + logging.info("task_prerun start.") + init_tracing() + logging.info("task_prerun end.") + + @task_postrun.connect def task_postrun_handler(sender=None, **kwargs): # Trigger garbage collection after each task From ed3bd01f212012789edeae81ee804cf2e20fa601 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 16:46:06 +0200 Subject: [PATCH 33/50] remove unrequired signals --- worker/tasks.py | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index e0aa36e..9b5747b 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -130,34 +130,6 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") -@celeryd_init.connect -def celeryd_init(sender=None, **kwargs): - logging.info("celeryd_init start.") - init_tracing() - logging.info("celeryd_init end.") - - -@celeryd_after_setup.connect -def celeryd_after_setup(sender=None, **kwargs): - logging.info("celeryd_after_setup start.") - init_tracing() - logging.info("celeryd_after_setup end.") - - -@worker_init.connect -def worker_init(sender=None, **kwargs): - logging.info("worker_init start.") - init_tracing() - logging.info("worker_init end.") - - -@worker_process_init.connect -def worker_process_init(sender=None, **kwargs): - logging.info("worker_process_init start.") - init_tracing() - logging.info("worker_process_init end.") - - @worker_ready.connect def worker_ready(sender=None, **kwargs): logging.info("worker_ready start.") @@ -165,11 +137,11 @@ def worker_ready(sender=None, **kwargs): logging.info("worker_ready end.") -@task_prerun.connect -def task_prerun(sender=None, **kwargs): - logging.info("task_prerun start.") - init_tracing() - logging.info("task_prerun end.") +# @task_prerun.connect +# def task_prerun(sender=None, **kwargs): +# logging.info("task_prerun start.") +# init_tracing() +# logging.info("task_prerun end.") @task_postrun.connect From b0f1533a6fa13a696ce8acc301ab63b5e3221baf Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 16:57:59 +0200 Subject: [PATCH 34/50] fix --- worker/tasks.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 9b5747b..050ad44 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -130,18 +130,18 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") -@worker_ready.connect -def worker_ready(sender=None, **kwargs): - logging.info("worker_ready start.") - init_tracing() - logging.info("worker_ready end.") +# @worker_ready.connect +# def worker_ready(sender=None, **kwargs): +# logging.info("worker_ready start.") +# init_tracing() +# logging.info("worker_ready end.") -# @task_prerun.connect -# def task_prerun(sender=None, **kwargs): -# logging.info("task_prerun start.") -# init_tracing() -# logging.info("task_prerun end.") +@task_prerun.connect +def task_prerun(sender=None, **kwargs): + logging.info("task_prerun start.") + init_tracing() + logging.info("task_prerun end.") @task_postrun.connect From 097490019c6aa08d26382f4d59729b971f250c4a Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 17:10:12 +0200 Subject: [PATCH 35/50] worker_process_init --- worker/tasks.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 050ad44..c2ba8a0 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -130,6 +130,34 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") +# @celeryd_init.connect +# def celeryd_init(sender=None, **kwargs): +# logging.info("celeryd_init start.") +# init_tracing() +# logging.info("celeryd_init end.") + + +# @celeryd_after_setup.connect +# def celeryd_after_setup(sender=None, **kwargs): +# logging.info("celeryd_after_setup start.") +# init_tracing() +# logging.info("celeryd_after_setup end.") + + +# @worker_init.connect +# def worker_init(sender=None, **kwargs): +# logging.info("worker_init start.") +# init_tracing() +# logging.info("worker_init end.") + + +@worker_process_init.connect +def worker_process_init(sender=None, **kwargs): + logging.info("worker_process_init start.") + init_tracing() + logging.info("worker_process_init end.") + + # @worker_ready.connect # def worker_ready(sender=None, **kwargs): # logging.info("worker_ready start.") @@ -137,11 +165,11 @@ def ask_question_auto_search( # logging.info("worker_ready end.") -@task_prerun.connect -def task_prerun(sender=None, **kwargs): - logging.info("task_prerun start.") - init_tracing() - logging.info("task_prerun end.") +# @task_prerun.connect +# def task_prerun(sender=None, **kwargs): +# logging.info("task_prerun start.") +# init_tracing() +# logging.info("task_prerun end.") @task_postrun.connect From dd2e9f844f4e008e9ce7b88977cfb32187c7580a Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 17:28:48 +0200 Subject: [PATCH 36/50] handlers --- worker/tasks.py | 54 +++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index c2ba8a0..4aea89f 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -130,49 +130,51 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") -# @celeryd_init.connect -# def celeryd_init(sender=None, **kwargs): -# logging.info("celeryd_init start.") -# init_tracing() -# logging.info("celeryd_init end.") +@celeryd_init.connect +def celeryd_init_handler(sender=None, **kwargs): + logging.info("celeryd_init_handler start.") + init_tracing() + logging.info("celeryd_init_handler end.") -# @celeryd_after_setup.connect -# def celeryd_after_setup(sender=None, **kwargs): -# logging.info("celeryd_after_setup start.") -# init_tracing() -# logging.info("celeryd_after_setup end.") +@celeryd_after_setup.connect +def celeryd_after_setup_handler(sender=None, **kwargs): + logging.info("celeryd_after_setup_handler start.") + init_tracing() + logging.info("celeryd_after_setup_handler end.") -# @worker_init.connect -# def worker_init(sender=None, **kwargs): -# logging.info("worker_init start.") -# init_tracing() -# logging.info("worker_init end.") +@worker_init.connect +def worker_init_handler(sender=None, **kwargs): + logging.info("worker_init_handler start.") + init_tracing() + logging.info("worker_init_handler end.") -@worker_process_init.connect -def worker_process_init(sender=None, **kwargs): - logging.info("worker_process_init start.") - init_tracing() - logging.info("worker_process_init end.") +# @worker_process_init.connect +# def worker_process_init_handler(sender=None, **kwargs): +# logging.info("worker_process_init_handler start.") +# init_tracing() +# logging.info("worker_process_init_handler end.") # @worker_ready.connect -# def worker_ready(sender=None, **kwargs): -# logging.info("worker_ready start.") +# def worker_ready_handler(sender=None, **kwargs): +# logging.info("worker_ready_handler start.") # init_tracing() -# logging.info("worker_ready end.") +# logging.info("worker_ready_handler end.") # @task_prerun.connect -# def task_prerun(sender=None, **kwargs): -# logging.info("task_prerun start.") +# def task_prerun_handler(sender=None, **kwargs): +# logging.info("task_prerun_handler start.") # init_tracing() -# logging.info("task_prerun end.") +# logging.info("task_prerun_handler end.") @task_postrun.connect def task_postrun_handler(sender=None, **kwargs): + logging.info("task_postrun_handler start.") # Trigger garbage collection after each task gc.collect() + logging.info("task_postrun_handler end.") From df24bfb7caf67e7741517f5d289b184829ed122c Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 17:44:48 +0200 Subject: [PATCH 37/50] handlers --- worker/tasks.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 4aea89f..e783356 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -151,25 +151,25 @@ def worker_init_handler(sender=None, **kwargs): logging.info("worker_init_handler end.") -# @worker_process_init.connect -# def worker_process_init_handler(sender=None, **kwargs): -# logging.info("worker_process_init_handler start.") -# init_tracing() -# logging.info("worker_process_init_handler end.") +@worker_process_init.connect +def worker_process_init_handler(sender=None, **kwargs): + logging.info("worker_process_init_handler start.") + init_tracing() + logging.info("worker_process_init_handler end.") -# @worker_ready.connect -# def worker_ready_handler(sender=None, **kwargs): -# logging.info("worker_ready_handler start.") -# init_tracing() -# logging.info("worker_ready_handler end.") +@worker_ready.connect +def worker_ready_handler(sender=None, **kwargs): + logging.info("worker_ready_handler start.") + init_tracing() + logging.info("worker_ready_handler end.") -# @task_prerun.connect -# def task_prerun_handler(sender=None, **kwargs): -# logging.info("task_prerun_handler start.") -# init_tracing() -# logging.info("task_prerun_handler end.") +@task_prerun.connect +def task_prerun_handler(sender=None, **kwargs): + logging.info("task_prerun_handler start.") + init_tracing() + logging.info("task_prerun_handler end.") @task_postrun.connect From f1b0cf9bc19f1f25e304207b56a979f7b49c865a Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 18:00:46 +0200 Subject: [PATCH 38/50] worker ready --- worker/tasks.py | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index e783356..f8e9eb0 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -130,32 +130,32 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") -@celeryd_init.connect -def celeryd_init_handler(sender=None, **kwargs): - logging.info("celeryd_init_handler start.") - init_tracing() - logging.info("celeryd_init_handler end.") +# @celeryd_init.connect +# def celeryd_init_handler(sender=None, **kwargs): +# logging.info("celeryd_init_handler start.") +# init_tracing() +# logging.info("celeryd_init_handler end.") -@celeryd_after_setup.connect -def celeryd_after_setup_handler(sender=None, **kwargs): - logging.info("celeryd_after_setup_handler start.") - init_tracing() - logging.info("celeryd_after_setup_handler end.") +# @celeryd_after_setup.connect +# def celeryd_after_setup_handler(sender=None, **kwargs): +# logging.info("celeryd_after_setup_handler start.") +# init_tracing() +# logging.info("celeryd_after_setup_handler end.") -@worker_init.connect -def worker_init_handler(sender=None, **kwargs): - logging.info("worker_init_handler start.") - init_tracing() - logging.info("worker_init_handler end.") +# @worker_init.connect +# def worker_init_handler(sender=None, **kwargs): +# logging.info("worker_init_handler start.") +# init_tracing() +# logging.info("worker_init_handler end.") -@worker_process_init.connect -def worker_process_init_handler(sender=None, **kwargs): - logging.info("worker_process_init_handler start.") - init_tracing() - logging.info("worker_process_init_handler end.") +# @worker_process_init.connect +# def worker_process_init_handler(sender=None, **kwargs): +# logging.info("worker_process_init_handler start.") +# init_tracing() +# logging.info("worker_process_init_handler end.") @worker_ready.connect @@ -165,11 +165,11 @@ def worker_ready_handler(sender=None, **kwargs): logging.info("worker_ready_handler end.") -@task_prerun.connect -def task_prerun_handler(sender=None, **kwargs): - logging.info("task_prerun_handler start.") - init_tracing() - logging.info("task_prerun_handler end.") +# @task_prerun.connect +# def task_prerun_handler(sender=None, **kwargs): +# logging.info("task_prerun_handler start.") +# init_tracing() +# logging.info("task_prerun_handler end.") @task_postrun.connect From 337a3862ce8aacebbce11a53cca5a8df0aa6d55d Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Fri, 28 Jun 2024 18:13:06 +0200 Subject: [PATCH 39/50] task_prerun --- worker/tasks.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index f8e9eb0..d865498 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -158,18 +158,18 @@ def ask_question_auto_search( # logging.info("worker_process_init_handler end.") -@worker_ready.connect -def worker_ready_handler(sender=None, **kwargs): - logging.info("worker_ready_handler start.") - init_tracing() - logging.info("worker_ready_handler end.") +# @worker_ready.connect +# def worker_ready_handler(sender=None, **kwargs): +# logging.info("worker_ready_handler start.") +# init_tracing() +# logging.info("worker_ready_handler end.") -# @task_prerun.connect -# def task_prerun_handler(sender=None, **kwargs): -# logging.info("task_prerun_handler start.") -# init_tracing() -# logging.info("task_prerun_handler end.") +@task_prerun.connect +def task_prerun_handler(sender=None, **kwargs): + logging.info("task_prerun_handler start.") + init_tracing() + logging.info("task_prerun_handler end.") @task_postrun.connect From 622f64dc12f3c554a223c931f1c5827c38a6f959 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 09:43:58 +0200 Subject: [PATCH 40/50] removed unused imports --- worker/tasks.py | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index d865498..e4a98bd 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -5,11 +5,6 @@ from celery.signals import ( task_postrun, - worker_init, - celeryd_init, - worker_process_init, - worker_ready, - celeryd_after_setup, task_prerun, ) from tc_messageBroker.rabbit_mq.event import Event @@ -130,51 +125,13 @@ def ask_question_auto_search( logging.info("FINISHED JOB WITH EXCEPTION") -# @celeryd_init.connect -# def celeryd_init_handler(sender=None, **kwargs): -# logging.info("celeryd_init_handler start.") -# init_tracing() -# logging.info("celeryd_init_handler end.") - - -# @celeryd_after_setup.connect -# def celeryd_after_setup_handler(sender=None, **kwargs): -# logging.info("celeryd_after_setup_handler start.") -# init_tracing() -# logging.info("celeryd_after_setup_handler end.") - - -# @worker_init.connect -# def worker_init_handler(sender=None, **kwargs): -# logging.info("worker_init_handler start.") -# init_tracing() -# logging.info("worker_init_handler end.") - - -# @worker_process_init.connect -# def worker_process_init_handler(sender=None, **kwargs): -# logging.info("worker_process_init_handler start.") -# init_tracing() -# logging.info("worker_process_init_handler end.") - - -# @worker_ready.connect -# def worker_ready_handler(sender=None, **kwargs): -# logging.info("worker_ready_handler start.") -# init_tracing() -# logging.info("worker_ready_handler end.") - - @task_prerun.connect def task_prerun_handler(sender=None, **kwargs): - logging.info("task_prerun_handler start.") + # Initialize Traceloop for LLM init_tracing() - logging.info("task_prerun_handler end.") @task_postrun.connect def task_postrun_handler(sender=None, **kwargs): - logging.info("task_postrun_handler start.") # Trigger garbage collection after each task gc.collect() - logging.info("task_postrun_handler end.") From 7ce28abbf6758830a85e2e9ab69c8ec33b99e341 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Sat, 29 Jun 2024 11:11:18 +0200 Subject: [PATCH 41/50] Delete coverage/lcov.info --- coverage/lcov.info | 657 --------------------------------------------- 1 file changed, 657 deletions(-) delete mode 100644 coverage/lcov.info diff --git a/coverage/lcov.info b/coverage/lcov.info deleted file mode 100644 index 3d54539..0000000 --- a/coverage/lcov.info +++ /dev/null @@ -1,657 +0,0 @@ -TN: -SF:bot/__init__.py -DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg -LF:0 -LH:0 -end_of_record -TN: -SF:bot/retrievers/__init__.py -DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg -LF:0 -LH:0 -end_of_record -TN: -SF:bot/retrievers/custom_retriever.py -DA:1,1,1+9wdnNK6ERHmSA06tdyPQ -DA:6,1,RDWQRIzCg9gLwFgbUhG0ZQ -DA:7,1,97oNYmi8xQccedqGxd27UQ -DA:8,1,Epduv66JxgjoxUaCT3nS2g -DA:11,1,s6T4UohrMBIsrcI68IgiqQ -DA:12,1,XJc56GRwdNlz7Pfca4ta2w -DA:15,1,FhljCPNPys8XCLWkwhX6tQ -DA:16,1,oQZL7x34ZmpY26UrXDNFmQ -DA:19,0,Qa1wQ2wVs41AUQxepYAgAQ -DA:22,0,mRVOQmWFDi50EtfArtMGKw -DA:23,0,ziMwbdP43Yl2rkv+Iw0+Vg -DA:27,0,/pJeNEIa+r0HHLINHDruGg -DA:28,0,GuHBXE7dyAR1Y0ynfbXBjg -DA:31,0,braB6j7cvq7N3Zid4KaqrQ -DA:32,0,5YC65MS1vK1RBYkxZkINoQ -DA:36,0,XlCerKuXPRvHHiX6G4l0nQ -DA:37,0,+4WCHe8YiFhx16gV8PHaiw -DA:38,0,FB660ofQJW2XNi3LECQSFw -DA:41,0,zpXzPHqwITv7OgzCdXjxQQ -DA:42,0,hqLZyd6T1Z9Dbx9cHMDy5g -DA:43,0,mj69/XSznAeubuOVDDQgCA -DA:47,0,36hUmZ4dEI4TxxoXjrxRWg -DA:48,0,GmJAjoIgF9rNAoVVXoryvQ -DA:49,0,4HIKapx7G/1BQUJ4xQuPlA -DA:50,0,qay3E+cJPb5xSdPWZrpBew -DA:51,0,v50IHDAPJiYZpVFrAPWUCQ -DA:52,0,ut8wnco13WKIzqf4F3XevQ -DA:54,0,xGgKt1DFNSwI4OqNYuCmqQ -DA:55,0,XPrAAc6YApLYKhPB2ugc5w -DA:57,0,bsKhhD3OVsNd7qpCNCVshg -DA:59,0,dSnd2LGkYzu2vL0jseg1aA -LF:30 -LH:7 -end_of_record -TN: -SF:bot/retrievers/forum_summary_retriever.py -DA:1,1,l4yWWF9Yn1gF6iRCnynHUg -DA:2,1,qgap8v00S1Ud7dKp+nEVgg -DA:3,1,ZwFry2q3ybNGgdFz5V3nNw -DA:4,1,S6WdzE5Nj4jcMIr2/RIeEA -DA:7,1,JZoyIbA0/Up5YJXsqDUYFQ -DA:8,1,4LgMgWBGno1n5MNJ8WRFsQ -DA:18,1,C1mbXdO4LpybcjJNbDJYXQ -DA:20,1,f+GuTgPLC7rYJZEnF8KkZg -DA:55,1,Rrf/KyYVoV7ZZGZVZ81NWQ -DA:57,1,/MYzux0mSuxPnCfDB/QApg -DA:64,1,x32FHFF9bNjommgpDHkbdQ -DA:66,1,HDI5TfzjszdVNEtxufbnyg -DA:101,1,exJXoRLYgNNdu03/ShoHrQ -DA:102,1,kUdJgUS+mcODFLg7Fk+OHw -DA:103,1,+f2nVTBEmuKiag075WVHkA -DA:105,1,osmxeRTnhC30Fq8zkebgGw -DA:106,1,0zgtOd30L7H3j7fx+w3Mdg -DA:112,1,K+yjYcF/QNzhfYmE7uZPAw -DA:113,1,9nMOpyz632hMtLnVoiQf0w -DA:115,1,dDsC6FL/ONgaG0AYHRlC5w -DA:117,1,x32FHFF9bNjommgpDHkbdQ -LF:21 -LH:21 -end_of_record -TN: -SF:bot/retrievers/process_dates.py -DA:1,1,EZEzNmL93PY96zsSuI0PfQ -DA:2,1,w4wzKKlLBqr27YH2UrIkBQ -DA:4,1,KA9tLtxJCmXGWOalg5c/zA -DA:7,1,+QD2wyYZcsC+Xvi5rjfg1w -DA:26,1,bFij5SG38wK0MWxmbT82Rw -DA:27,1,iBjZwelGAT6u9wcCT5RkQw -DA:28,1,K1+sxVnEq9jO8BqcJwVAZw -DA:29,1,pf92umTBWmtiEB1EpuczhA -DA:31,1,2uLiygcEABFP5hS6AeQwHA -DA:34,1,tYltTNWket1uI7UNICnqmw -DA:35,1,RwimGdhYyabPJL4e+Fvqnw -DA:36,1,LTTh/+NrsNUrIQkwnwP3hA -DA:37,1,Da4uV101pljkvITvb/orrw -DA:39,1,p3JK0mYGwzpZUPpV9FxrSg -DA:41,1,7mawqEbVi5mg0yRWqaqupg -LF:15 -LH:15 -end_of_record -TN: -SF:bot/retrievers/retrieve_similar_nodes.py -DA:1,1,usL6GpKXVxVT8icw7bX7uQ -DA:2,1,nuxjJx40cdYGSgrgJ50F1A -DA:4,1,KA9tLtxJCmXGWOalg5c/zA -DA:5,1,vMmpgxZJRuue6rFB1P0BSg -DA:6,1,qgap8v00S1Ud7dKp+nEVgg -DA:7,1,ZwFry2q3ybNGgdFz5V3nNw -DA:8,1,XJc56GRwdNlz7Pfca4ta2w -DA:9,1,IDxH3gjD3VSVrI4xLmrtQQ -DA:10,1,YQAIr2ESlBeDuRf0TeEF5Q -DA:11,1,2PHsUT6zqyYmFXFvARFHDg -DA:12,1,S6WdzE5Nj4jcMIr2/RIeEA -DA:15,1,IPFoc7JFmfs5ZGCvDdLKww -DA:16,1,mqoJzotpXwlnL7jkyBBK1w -DA:18,1,4LgMgWBGno1n5MNJ8WRFsQ -DA:25,1,GeCb9dZfUzto8bJsxLM/OA -DA:26,1,BOdqpSj22HMpl9h/3UQEnQ -DA:27,1,EEPdD4zi1M7G+hCfGc7kFw -DA:29,1,d+ncuJjKr272GS184mU5Mw -DA:64,1,4lzkDJLlQXSyBjO/ZBZ0tg -DA:65,1,R2Ju8xaiGLrApGDnnYA2Cw -DA:66,1,sXjQBJruC0xTKR4b+u8JIA -DA:67,1,pWP8puMbrWzJ9jyYSNijYg -DA:70,1,1DaXT/QWnDNNMkOiYOOCAg -DA:72,1,M57N117IoUfJdZ4DKcJp9w -DA:73,1,ACylfUs9k82fbh1JUVpkFw -DA:88,1,i9tYTGC/B+qp4Thp+0XtPA -DA:89,1,eJXGGDxmpnTlPApZ05Mm8w -DA:90,1,bkIDFrOGWBbeePkt6w8EJQ -DA:91,1,IDhEFBr50D3yQ1PgRwIPoQ -DA:95,1,DVM8lfdtc7XvpdnChfyzZA -DA:108,1,NYgySTsjOvX0beJFX6yMkQ -DA:109,1,87A78EES6C0LtBxGYDZtBg -DA:111,1,Dza3guFRyWWcXuaktAvUUA -DA:112,1,42k3Krouu/7c6z6xkNm7jg -DA:113,1,74cSsiTs5tJX4LFRiR9Z/A -DA:114,1,NcFv4vwj+pjug1e2Bo890w -DA:115,1,Bp6QbYpwe4PVOg8lbJA/hA -DA:116,1,jrIPKZt5zHATQ3k7gB0zMg -DA:118,1,3X+u/Gx10tHDgUlKjZ9leg -DA:119,1,enK82Bd/RAeIx81NTDQ/7w -DA:124,1,AEFyjNRKgOrooYQ/nYO/gg -DA:127,1,unkkFkVkr+wa6RT5uE8c1w -DA:132,1,HJEWWSze80ShpmLhx/FNmw -DA:137,1,SkkggBMbU32rJ3pNviGxAA -DA:142,1,XDQoFg/N3OcYqb/J4NJzUg -DA:143,1,QAy+2Mz9GQWA6hn/DfyOnw -DA:156,1,Ap+b3WevKixvNInY8BOstQ -DA:158,1,M4Wpq7+IHVDAhVO1Q+IA+w -DA:160,1,tzZodAsv3aFszptj+1OcnQ -DA:161,1,r+V3a23Womuti9fftxA7Lg -DA:165,1,e8upNNRAcwfVSEfPvxiucg -DA:167,1,FIGBJFFS77Q/1c40b8di/A -DA:168,1,NdO+Zxe21pmF6dXu3zWdMw -DA:170,1,UkwhkGz1XXrgDOp60bursg -DA:171,1,Cj+UwANUqSKqiqpqgnF9hA -DA:173,1,X1yR/f77a2wYrip3qWXYBA -DA:182,1,k9pTjjD2vJTvE7PyzhXJPw -DA:183,1,72SNpwVsw/o9etOCZ3VLaA -DA:184,1,5M8S6frycaNS3uQnlnMr2g -DA:186,1,3WsKRpW8oQhkQv0F14ak/w -DA:190,1,mUbbKvPqWHojLjV0v95bwg -DA:191,1,DdHNUt5ovAV1XQYsiT8AiQ -DA:192,1,qay3E+cJPb5xSdPWZrpBew -DA:193,1,v50IHDAPJiYZpVFrAPWUCQ -DA:194,1,siWxFuIgfjXgP30tTFf/aQ -DA:198,1,xGgKt1DFNSwI4OqNYuCmqQ -DA:199,1,XPrAAc6YApLYKhPB2ugc5w -DA:200,1,Rx8CNN7oYnLYIPSkwQ5G2w -DA:202,1,diWaG+5ccHGPOYFR5G1hZw -DA:68,0,H8roOaxT/dA+ek63CQkNsg -DA:121,0,genHDsC84NGAm0Lr+up+qQ -DA:145,0,I/UvgUIdm1YWL558qpxsAw -DA:154,0,m+4mRvWfRixXNFnr5PmwsQ -LF:72 -LH:68 -end_of_record -TN: -SF:bot/retrievers/summary_retriever_base.py -DA:1,1,3H1a4ALTm6+CsKOSplE0HQ -DA:2,1,qgap8v00S1Ud7dKp+nEVgg -DA:3,1,RdpxTAbMfKRSNi6jiLRhow -DA:4,1,S6WdzE5Nj4jcMIr2/RIeEA -DA:5,1,r/vTV1phZQGHkBIJdSWs9w -DA:8,1,uUFp+g6+u1KGwCpCapD0RA -DA:9,1,4LgMgWBGno1n5MNJ8WRFsQ -DA:36,1,+Cv21TkBoI5AI2DIkaXSUQ -DA:37,1,IMKXrfpxE/NMfZh6Hz035w -DA:39,1,giUPm3pCLbXkU0PTyH+bFw -DA:55,1,anaPnxi7G3x+uzrWDKUlIQ -DA:57,1,rosmnQjUb4d/mw28S/QFyA -DA:59,1,Gh3uqVxTsog8Q14gAbyV5g -DA:60,1,MTyrxoRbX0cdVfuQK+cdog -DA:62,1,5M8S6frycaNS3uQnlnMr2g -DA:64,1,8oWHAROlVU38/+SOWIYZ3g -DA:74,0,aTZE5skweNKnLAkkNEpaQw -DA:80,0,6m2c6rMWrwACEmomupZfAQ -DA:81,0,ItkzfojqvwK4mDljHSDeZQ -LF:19 -LH:16 -end_of_record -TN: -SF:bot/retrievers/utils/__init__.py -DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg -LF:0 -LH:0 -end_of_record -TN: -SF:bot/retrievers/utils/load_hyperparams.py -DA:1,1,7Z9Lj4ed27Wf2hBX6jooEA -DA:3,1,oBn3m+aX07XSXyLZ7hhJvQ -DA:6,1,pALyYLvuXgvTjbXn92fkbQ -DA:21,1,uMN5/uIZZnp9nEAOplwhrg -DA:23,1,E0RQB+VgBEJ4QKNtxT2ZCQ -DA:24,1,obtvc0wXNzMHpam/R9pARQ -DA:25,1,xhBQ4Bxy/t+hksYxRkBLMg -DA:27,1,EevRcw2hn5gowSBZyczEKg -DA:28,1,oBoY/n2VDmZk3l8eOirLDg -DA:29,1,EsZQAiXrpsXDNrw17z9J5w -DA:30,1,HZ54ZIWn+eFEat0+ZEm5cg -DA:31,1,tl0+/G75a26GKwswbSpW7Q -DA:32,1,rGIsTu9A/YXl9CkiXRexpg -DA:34,1,zw4+yRAywnofajwDMr1yLQ -LF:14 -LH:14 -end_of_record -TN: -SF:utils/__init__.py -DA:0,1,1B2M2Y8AsgTpgAmY7PhCfg -LF:0 -LH:0 -end_of_record -TN: -SF:utils/credentials.py -DA:1,1,7Z9Lj4ed27Wf2hBX6jooEA -DA:3,1,oBn3m+aX07XSXyLZ7hhJvQ -DA:6,1,gvBB+eqYJmZEE370I4PLJw -DA:20,1,uMN5/uIZZnp9nEAOplwhrg -DA:22,1,YJqLCDyWmMT4Vt80XKej7A -DA:24,1,IxExvRJQkW08pwLjPeV+AA -DA:25,1,saUrv7XldTgMIzkqBUF3sw -DA:26,1,n5Fd4k6CRQXcqYOJ1AroeA -DA:27,1,5aSjAewLt9BgZu1M2CyLDg -DA:29,1,UrYRi5Ku4cGH9QytPbwuIg -DA:32,1,Cm8bLUg146j1KAg+xEigfA -DA:46,1,uMN5/uIZZnp9nEAOplwhrg -DA:48,1,r0VtMIE142HhS1QHUy84/Q -DA:50,1,sJYVqUX9JungmATURLwWTw -DA:51,1,7kGLhyj2Gz1FgT9D0bb9Vg -DA:52,1,+GdQ45oCl4mUsGv9HojWfg -DA:53,1,e710G8kzqt5PjHHCzkMTng -DA:55,1,H17V4FJl1OEtok8qC+q/+A -DA:58,1,kSnEDTyHvsGabszAxYqSww -DA:72,1,uMN5/uIZZnp9nEAOplwhrg -DA:74,1,/vIX4NRiJ6gfFdtd7v/5CA -DA:76,1,LnlORR7UjQKf3WTSqC+LpQ -DA:77,1,l8Cclrt4B4dMSzJot1zPcg -DA:78,1,hIKLCkB8Re7lOm0QLJbg6A -DA:79,1,y1aZfDdlZokMd4D/4FAmwA -DA:81,1,T7zGOSr3oGWrSM/CzDYDcw -LF:26 -LH:26 -end_of_record -TN: -SF:utils/data_source_selector.py -DA:1,1,OXA5GZhWrVnCdJ6JVjaTww -DA:2,1,6XxZigYIXTEoXqTlBNKh4A -DA:5,1,cq+6nGZ4L624r1Qu2pX9IA -DA:6,1,bDTSD3NwJzYLX9VLOWNrog -DA:21,1,OfPQ5nOrizIBxrqsb+18Ww -DA:22,1,khL+YRyaqSGXvXfJuXRwfw -DA:23,1,9FG7XLm0rLCGHwL0YJNB/w -DA:24,1,j1mUH+lpTuJQfouULXJ7Kg -DA:26,1,fEBuiKF4P+NVarquP1gmjQ -DA:27,1,QFftjygbzu3/aoWrwPWGhQ -DA:28,1,tNHEj4JtLDnMXIsuX0QWgA -DA:37,1,1GeTxQg5T4sfLOBh5IROaw -DA:38,1,HVyrBJ61ni9ux6MdUIeaUA -DA:41,1,pK1ZUnMvS6wV8z2nLw6pKg -DA:43,1,wzxE+BQWKgUayLEFKTCYmQ -LF:15 -LH:15 -end_of_record -TN: -SF:utils/fetch_community_id.py -DA:1,1,cJoQvc8d9CEP56sPh71adA -DA:4,1,lmwdGMkCHlWXv2nNm/Uxdw -DA:19,1,YjVJLe7hbK7XTWsOoFVj/w -DA:21,1,QA9dGGPTeIp07ShApLAF4A -DA:24,1,U4Sjk6MLNWi4lp3eZMYMcg -DA:25,1,A4crhuYlU3KiAJlkPHK9QQ -DA:26,1,nqKcow6NrIpARXr+QJdOSg -DA:27,1,QKuzwW865EXj8VNYNgSC9w -LF:8 -LH:8 -end_of_record -TN: -SF:utils/mongo.py -DA:1,1,EZEzNmL93PY96zsSuI0PfQ -DA:2,1,gGTQvjsCFZRTAK01JA8k5Q -DA:4,1,aQa914DmkZxIUR93SEZpuw -DA:6,1,7wOattRMH258CtQs+UrpsQ -DA:9,1,yl+B0YFGI3t/m+5Suhwqfg -DA:10,1,83sxBcOsVIr2WiCqCQbLZQ -DA:12,1,gFdnwIzxJ64X+nD2UauP7A -DA:13,1,nedGLtP1l6hUktBMlnMhYA -DA:16,1,9pf1HZayXGTtSA9Rvl+WMg -DA:17,1,quD3RtvYZgmdln2jYSS7tA -DA:18,1,CUeVzuv4ZHDgYpULPWrh+A -DA:19,1,p8beMpKHz8de2xpXvL/QrQ -DA:21,1,KxgaGtJPkScc4OG7mJ59vg -DA:22,1,UE5ijQ2kRfIL7H25RlfGsg -DA:23,1,/k6qkyUHhH0yi+zpCWRQEg -DA:24,1,atc/IsM8O08RY1DyTj03hA -DA:25,1,xvWIcpcyNr0gyGcTkdMxXw -DA:26,1,wqqc+RCz9O1/BPQQ/avpsw -DA:27,1,pqGPD3HqI8BK+o/OexZo/A -DA:31,1,kixoR3OQ3VPgFNcOZnJMCw -DA:33,1,BN/xY+w6ECzfgEoHXowADw -DA:34,1,fgy3JEj8zwF+XOMFGgrcLQ -DA:37,1,ta6BcHtLzbVui7kSYavGHg -DA:38,1,hsfmwwQPyB4q/pQbrCvugg -DA:39,1,XDsjAHuILiKaPifDcNRk/A -DA:40,1,OLNS5e2ODcauq2phB8fr4A -DA:41,1,D4zo9qN1NT7YrDJBIRiK+A -DA:43,1,IBgmfx3PCWZ6RwfdTKp+Fg -DA:45,1,E2z+ad+nCvPlAX1NpKcGVw -DA:14,0,+QJTRb5rAKExUUr0WnnqAg -DA:28,0,LFZOE01yWzhpu4V3LUU0kw -DA:29,0,KQd83bxFJN5wqRs65ZHiAg -LF:32 -LH:29 -end_of_record -TN: -SF:utils/qdrant_utils.py -DA:1,1,ntJL14mrBgAly/E3tslOUg -DA:4,1,t9LFvlhmTNDcbcR06swJFA -DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ -DA:15,1,bxcqzIMUQ67BtZZ0H64HWQ -DA:16,1,lwSaYcYgqD6duWfx8iDyCg -DA:18,1,o9EuHdH+gpLJ8BZ0vpjMdg -DA:32,1,RI9Ka1LT1e/iffNnfq+bLg -DA:33,1,flM9IHJmqBjqVddLMZTPMg -DA:34,1,nAYiUkgpfqrBg3mJ07kzXA -LF:9 -LH:9 -end_of_record -TN: -SF:utils/query_engine/__init__.py -DA:2,1,Vnj8yLJpJ/wVehtFqro6mg -DA:3,1,Yen9HUSDB0Wd67+ldfKlng -DA:4,1,/+xElf2yXN2V7g/cx46R4w -DA:5,1,+faw0bsltOVhGJ4yjZDL9Q -DA:6,1,d8GL/JuynocTO/XRp6VV+Q -DA:7,1,J4oH3WJqtVVhhLElWxqk+A -LF:6 -LH:6 -end_of_record -TN: -SF:utils/query_engine/base_pg_engine.py -DA:1,1,VRsPke+iz1KOD5Ylfx/vow -DA:2,1,tv1rZhcNs3rWaZFnKxX8AA -DA:3,1,9zqOVuK2+mKhEfPQ8X750Q -DA:4,1,i4oCYN8ZAW8fFZiqSKryuQ -DA:5,1,1rLMUIEu7V96D7PKPUCUeQ -DA:6,1,wkfOwWcqhgnKREP2SoBl+A -DA:7,1,r/vTV1phZQGHkBIJdSWs9w -DA:10,1,1bcURF390sME4Zf7IEeg+A -DA:11,1,H8dNfXkovyjk0TB6ywYU0w -DA:25,1,NcYJVT1t86PY5bH1L4WMag -DA:26,1,lwSaYcYgqD6duWfx8iDyCg -DA:27,1,8CZJFonIL+691rDOKQSd8g -DA:29,1,LYmpQR52gMFxj5cQAgvBmw -DA:44,1,xOFEHQFRGWy9u/WDYjQNqw -DA:62,1,bAiir3mXHBEtXikoRFEOPw -DA:63,1,g/SCiHtEQsEcR+wXlI7gqQ -DA:66,1,C6ZtZYzya5SESpmY0jNCRw -DA:67,1,/NgtN11YZmhPFCQOzPd8bA -DA:69,1,DougZ9SUAHum4wQgjTyO4w -DA:71,1,xJlRV8Kxyu7FST2sQDK4bg -DA:73,1,v6Z2he5grryd4Z0ffKpgOA -DA:79,1,oaD/kc+MzLZ0/CzfGaJX2A -DA:80,1,ItkzfojqvwK4mDljHSDeZQ -DA:30,0,s4JEvRyP+y43QrQVUGI17A -DA:33,0,kihnT3XHavhUNbNr4ho+6g -DA:35,0,VBOpcm7qtCU0WDTcl8sXBw -DA:38,0,iXXJG8Mlu5JagixPsayHsw -DA:42,0,rRmN9Dhy/+TqccGt2h3m4A -LF:28 -LH:23 -end_of_record -TN: -SF:utils/query_engine/base_qdrant_engine.py -DA:1,1,tv1rZhcNs3rWaZFnKxX8AA -DA:2,1,9zqOVuK2+mKhEfPQ8X750Q -DA:3,1,Epduv66JxgjoxUaCT3nS2g -DA:6,1,1rLMUIEu7V96D7PKPUCUeQ -DA:7,1,6yg1SvuqaHW9gVNVzNYVrg -DA:10,1,HYj/N35l9bZAx7FsPcM3Zg -DA:11,1,H8dNfXkovyjk0TB6ywYU0w -DA:25,1,NcYJVT1t86PY5bH1L4WMag -DA:26,1,lwSaYcYgqD6duWfx8iDyCg -DA:27,1,nHZNxusJFY2cf/fBKlEIRw -DA:29,1,LYmpQR52gMFxj5cQAgvBmw -DA:30,1,3x3ebQUpyH0B5GozBlXRsg -DA:33,1,kihnT3XHavhUNbNr4ho+6g -DA:35,1,y/eQGOxxhvfE76tIHc6cwQ -DA:39,1,iXXJG8Mlu5JagixPsayHsw -DA:43,1,rRmN9Dhy/+TqccGt2h3m4A -DA:45,1,xOFEHQFRGWy9u/WDYjQNqw -DA:61,1,cNSMxn65O/ZzNyMdX/7n0g -DA:62,1,o0upLLxBXoxpEIXpEFDjrw -DA:66,1,ISnP9S4BsPcPJGVvKfraWA -DA:67,1,ItkzfojqvwK4mDljHSDeZQ -LF:21 -LH:21 -end_of_record -TN: -SF:utils/query_engine/gdrive.py -DA:1,1,C8DhKIOrNkqmLLr7blG7vw -DA:4,1,66+KyTBbTDXPKLFpM19sEg -DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ -DA:6,1,/Hs5KWsUByIj7S7tOxnUAw -DA:7,1,QC64UFI8neRNXWDeZEm1Zg -LF:5 -LH:5 -end_of_record -TN: -SF:utils/query_engine/github.py -DA:1,1,C8DhKIOrNkqmLLr7blG7vw -DA:4,1,ZADGWypDUyWGLHxGn9Lzog -DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ -DA:6,1,+B610GO5jHoGxR6V4GkacA -DA:7,1,QC64UFI8neRNXWDeZEm1Zg -LF:5 -LH:5 -end_of_record -TN: -SF:utils/query_engine/level_based_platform_query_engine.py -DA:1,1,EZEzNmL93PY96zsSuI0PfQ -DA:3,1,nOgG6xCCb4HBDSN3x9QS5A -DA:4,1,xeVnH2DNgPEjUcwgJVQehQ -DA:5,1,tv1rZhcNs3rWaZFnKxX8AA -DA:6,1,3H1a4ALTm6+CsKOSplE0HQ -DA:7,1,DDgKR+7enopNKdkqI8I62w -DA:8,1,22ORhkU2P3imK0vCquiKNQ -DA:9,1,WKJgOHAAbKxKt/3sZtK63g -DA:13,1,7FwFHH3k33riE6ReKUYWUw -DA:14,1,ZwFry2q3ybNGgdFz5V3nNw -DA:15,1,DCDK4Yl+/ELe6m+70OYJ1g -DA:16,1,gIY0rCnKSLWVSRd76jn1Mg -DA:17,1,sJebj/U8hMCZj5D1eM0aJg -DA:19,1,YjzuKhn/xRwHSBjOcQhsdA -DA:31,1,aAaxbH5EhJE64W9zBhIF9g -DA:32,1,3Yt4nvHJdyo2mqZ49Tnj5w -DA:33,1,wtC4abIbxsngso+ns9k6vw -DA:34,1,kAXWcSibUmRxDnaE5g0mJQ -DA:35,1,lyWcrb86KF2Hcx929BzmxQ -DA:37,1,p2Qx/bziJstmcY6BKOsgBA -DA:56,1,Nnqu9X+lU2lagW39QNhFWA -DA:57,1,aQZXZHzGnOZY9rrOd6/wvQ -DA:112,1,+AoWudgyI70lgbtxX7lqig -DA:114,1,5YOAMYmqtQNtDb2irXQR0w -DA:117,1,y55N4o1rvRJyLK/0r/5L7A -DA:118,1,WXOqRntq6K1mGRBen+UcbA -DA:119,1,zBvKWW8jFIhbdTFZsnTXVA -DA:120,1,lLRy3eVWI8Pt8RNaSKBsug -DA:126,1,TC5ZkNxN5KKk7z8DTijqjg -DA:128,1,O8d2LBsxkTL8X4P7tH1g7A -DA:129,1,r9RgqFbQ/+jUw8oOkCWalw -DA:138,1,mkH3WvSUvL2/3P24Hu7cKA -DA:139,1,h4T9Vvca+5tmqAtyyyXNyg -DA:141,1,tDdqYg3dq/wD/tVD8vu9/g -DA:143,1,KTtT/NxaurUIwrbxcvtcaA -DA:144,1,yKiHkJzAUkqGbTUr32ub9Q -DA:145,1,kj6Z4kNQl+AXmKR+j3zAYA -DA:147,1,nuShR7aDtL9Tmh7gKu9FIw -DA:154,1,Nnqu9X+lU2lagW39QNhFWA -DA:155,1,lafnpJ+CQMAdI8iasAI41A -DA:198,1,+AoWudgyI70lgbtxX7lqig -DA:199,1,c6q+PcdvoDVhU0DNZ1PvZg -DA:201,1,8rFzc+DLQVEr0zGZDWPbCw -DA:206,1,IYUyGR2ggWDMKtKUakHiKw -DA:209,1,7JZtaXtmzqywgswRvQt7+A -DA:211,1,jepe1mImdRVnsx4Js59XKg -DA:216,1,58c+JAyIdGn+tNpgL+tKQg -DA:219,1,J19yyj1rBA+MjKawC6oHOg -DA:223,1,V/hzcQrmu7D3FUS+Uvd8tA -DA:229,1,kn8wpgTeUaT9YyomopUMhA -DA:239,1,evg8qzxYp7Is+U7oBQ4HWg -DA:240,1,izA1BjWd1igKsXjjoKSKQw -DA:244,1,C6bghf8IPyZTFyYwr+agEQ -DA:245,1,9ztu/13sn+eyjgOcCfcsGQ -DA:246,1,J6uEf5KSYdEBC2a35vgBfw -DA:247,1,zRB9nprox4hkMNnasm1DfQ -DA:248,1,h4T9Vvca+5tmqAtyyyXNyg -DA:249,1,0fJTQHj5cDj1/stYiH+PTw -DA:251,1,HJIEUG2cMkYCGEwwS43VRw -DA:255,1,PH3ft6eUMPDGM/KLBdxVvA -DA:262,1,sgXKOyybp4yEVrJLhkOkkQ -DA:264,1,SHylApe6S33xsDgn0BMOCA -DA:270,1,hRgQD5up9E4XZrIkKN425g -DA:272,1,U9dLMFnDTp4r1/r38zkvng -DA:273,1,VdJ5KSGdN1AmU/IO2DW8vg -DA:277,1,pu4uDADWR92AgQ4lElnXng -DA:280,1,C87Lpztr4KKJLFNDJA1OTQ -DA:308,1,9/pmpwUZHacFljsBY7h3sA -DA:309,1,at/HVooxpizTsAJ4uAUG7w -DA:314,1,KXVUWpE1ypka5ImuW1QMpA -DA:320,1,VH4nv2g4YIj+iGTWpSQYIA -DA:323,1,LrdimJrx2agAR/ZKdZ1A9A -DA:344,1,kXJz5rsWvT06VgqAoVleRw -DA:346,1,uBcLKYMt4YuMysNBii+5yw -DA:40,0,jepe1mImdRVnsx4Js59XKg -DA:45,0,mHs0z5vDWZxuReGU6yh4VQ -DA:49,0,9+0wI5fq5LfHNN4soQijpg -DA:50,0,GkNF20k0ddXAQjklAJZQmA -DA:51,0,b9ewJXUvjgWBkQaDeA8l8w -DA:52,0,ty2ljlDXsDfuBiNGV+Op0g -DA:54,0,KPoxs3J/EBRSZ4FRpyqT9Q -DA:242,0,9FqK6DjAyzNXTmh2W+ebfw -DA:281,0,FjBQWq9IG4J6Cm7RfALnWA -DA:286,0,Gz5HHhWWOcMzaSURTUFBUw -DA:294,0,at/HVooxpizTsAJ4uAUG7w -DA:297,0,9/pmpwUZHacFljsBY7h3sA -DA:298,0,KXVUWpE1ypka5ImuW1QMpA -DA:304,0,VH4nv2g4YIj+iGTWpSQYIA -DA:324,0,6ChizqLUbucwiNDDkJLrDA -DA:328,0,5lqBTTJ5h43YzUS8pQ4tog -DA:333,0,eZP1y6iLmUAlnhAelnCupw -DA:336,0,5YBDint/C2L/36Y30bS/pg -DA:342,0,lMU2/UJ7v9WOAuw2dFSC7A -LF:93 -LH:74 -end_of_record -TN: -SF:utils/query_engine/level_based_platforms_util.py -DA:1,1,EZEzNmL93PY96zsSuI0PfQ -DA:3,1,KA9tLtxJCmXGWOalg5c/zA -DA:4,1,ZwFry2q3ybNGgdFz5V3nNw -DA:7,1,1n0+DB+0GKrFV9FTWNebSA -DA:8,1,2rHnDUvBBQWvCCEwdSz4mw -DA:9,1,PeQpl/2nVKsQpLWJOzEqQw -DA:10,1,zRgFE7FqhvZdMz1X95SX7Q -DA:11,1,GHz25t/A39Sr/cnYnniNCQ -DA:13,1,HNZxq1ze/z5tqRDtNZKTdA -DA:20,1,UBWWkmgHPHVfsMCI1Fsajw -DA:38,1,uBcLKYMt4YuMysNBii+5yw -DA:40,1,3z0QwSAFw5zRa9xwH+rgEA -DA:62,1,eLOsbqmnKW0+8kNbMCMDLg -DA:65,1,osmxeRTnhC30Fq8zkebgGw -DA:67,1,qZMvO0jjLdJVah0rd8S/jw -DA:68,1,Qkf8eNyjQqG0IH7RqJQ3+w -DA:69,1,PY15Nh0HkYgRKH6u93BSJQ -DA:70,1,68Yt75DazluknIc6D2DJXw -DA:73,1,GmbNDixOUUQ8VA997/7nbw -DA:77,1,mItPeXzAYUKXS9vXTJTWYw -DA:79,1,IsCQ9tgCFPPgq6WnIOsimw -DA:81,1,DvnMWQ7HgHURrxiyFIgH4g -DA:99,1,hRgQD5up9E4XZrIkKN425g -DA:101,1,9dqwsZzGUS6Yl4TftdS6GQ -DA:103,1,gJ439ucMTM4q5oT2Tov/Lw -DA:107,1,e440eUgPColUOy6pY34qVQ -DA:108,1,2SGzQDtqdvV4c37zEjo8Mw -DA:109,1,YSXhRM4A5II45l7DOsPJzw -DA:110,1,fPZKnSTL7HTWx6ZDYv5gLA -DA:113,1,rf+yiaZsFd2skz8OTryeUA -DA:118,1,sRbiZiQQFviDtVPK4JvfLg -DA:119,1,n6gf1L4xPtondA2Fo95wQg -DA:125,1,jqi+MeL/wMiw3ev5TkYnbw -DA:127,1,zJ63EizQBDp4hcpXusKZkw -DA:128,1,oNfSos/b+wErmWh2QP+c8A -DA:154,1,JnuqTw09b0rXs7pyJ5g2sQ -DA:158,1,q8024D2AVZGkIsSbhHUOdQ -DA:159,1,pLfMADHw+eGw8ECl0Z5aSw -DA:160,1,E8AYJN10pVbhOioFIMeHAA -DA:167,1,RczncNMSMdyFin4BxazGXg -DA:176,1,XTmIht7CQjYJLiUKNyewJQ -DA:179,1,kIUZ7J+O00xTJkiiIyNlaA -DA:183,1,UVbveT4931BLF02gVy8O/w -DA:184,1,noAL/2Aq9wHOV6+GxdUZzw -DA:190,1,F4xCoijosIqQAkElo3fWEg -DA:135,0,gFvTFqI1qeC4aC0n/wkg7g -DA:138,0,oNfSos/b+wErmWh2QP+c8A -DA:145,0,Y+9Cpj0wdzCrYKkYoQEr2w -DA:147,0,oNfSos/b+wErmWh2QP+c8A -LF:49 -LH:45 -end_of_record -TN: -SF:utils/query_engine/media_wiki.py -DA:1,1,C8DhKIOrNkqmLLr7blG7vw -DA:4,1,NRapDFy+V5xvFfDLz5AwKw -DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ -DA:6,1,0DvXG5U+Mac124OTxMZ50g -DA:7,1,QC64UFI8neRNXWDeZEm1Zg -LF:5 -LH:5 -end_of_record -TN: -SF:utils/query_engine/notion.py -DA:1,1,C8DhKIOrNkqmLLr7blG7vw -DA:4,1,taUtbU69r/u+KwuhqbZXwQ -DA:5,1,NBGY7hW3zO9IDJGSxYPIgQ -DA:6,1,WApUn0uoi5HwYH8FxulgWw -DA:7,1,QC64UFI8neRNXWDeZEm1Zg -LF:5 -LH:5 -end_of_record -TN: -SF:utils/query_engine/prepare_discord_query_engine.py -DA:1,1,FI1BbsECoGeHqt8kW925Kg -DA:3,1,O/Qn0s4Cmfehw44CStHXEw -DA:6,1,hvsmyd45Rodf/tBfNPWgPw -DA:34,1,q+12rfq5jrwpiWDQodqLhg -DA:35,1,eQava1okCaI1vmj/z0O7vw -DA:41,1,v4mNTOz+svYfaDNdHA++JA -DA:44,1,4OTfIyNPew3rgYQKc0Uofw -DA:68,0,oXzDsL38DFI/SF5Wkrqijg -DA:77,0,YRW/xAcZ0a28FDbybpQOAQ -LF:9 -LH:7 -end_of_record -TN: -SF:utils/query_engine/prepare_discourse_query_engine.py -DA:1,1,FI1BbsECoGeHqt8kW925Kg -DA:3,1,O/Qn0s4Cmfehw44CStHXEw -DA:6,1,SJPP3rgXO4WKqYx8e9xnJA -DA:31,1,q+12rfq5jrwpiWDQodqLhg -DA:32,1,eQava1okCaI1vmj/z0O7vw -DA:39,1,v4mNTOz+svYfaDNdHA++JA -DA:42,1,ts/D49naBdnJYvavhOMruw -DA:65,0,oXzDsL38DFI/SF5Wkrqijg -DA:73,0,YRW/xAcZ0a28FDbybpQOAQ -LF:9 -LH:7 -end_of_record -TN: -SF:utils/query_engine/subquery_gen_prompt.py -DA:1,1,OaTlemF3bXMi5MP7Db8vkQ -DA:2,1,da4F/HJkmdp35fCZoBYhug -DA:4,1,0+n0VDq2WJj6tgatFS0N1A -DA:5,1,p7jXd7VVorunNziWj6uTSQ -DA:6,1,3qjx/WnOrCQjYiF4JZTaTw -DA:9,1,JbdHvfcPRGbdQrtxirTU3Q -DA:10,1,GO/ZdycP15VWEePnEr38TQ -DA:11,1,O0J31uRmy1IE8NTEkT6wjA -DA:12,1,xUsGrHzeZSrjujDYvG8rqQ -DA:13,1,yqdV1UNZupsu1vbhHyQTNw -DA:16,1,YShAQ9o49XB7DbWwXU+SiA -DA:23,1,olsSoI2ZhROpde/oLnsWsA -DA:27,1,hgSlxMu28mzot0TyCC/FXA -DA:37,1,oH4MclLKdozptQeZO7CXFA -DA:38,1,iNdFOle/XUxgJDNuyCqOpQ -DA:46,1,PKlGrs30MCJpf45GqcuU3w -DA:48,1,FOJMonFW9MigPvGx3lVkRA -DA:70,1,d5ujy6JQKRcuZhFUBFDqMA -DA:83,1,K/JLImFIZiBQYyJ5dS7jTQ -DA:84,1,rU3AJeD7nrlk8aGyggdiXQ -LF:20 -LH:20 -end_of_record From 34dac4feaf27d06eca3ca9928da3fd4a37842f2c Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 11:14:24 +0200 Subject: [PATCH 42/50] imp traceloop --- utils/traceloop.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/traceloop.py b/utils/traceloop.py index 9c088ba..4b1c1c9 100644 --- a/utils/traceloop.py +++ b/utils/traceloop.py @@ -7,7 +7,9 @@ def init_tracing(): - logging.info("Initializing traceloop...") otel_endpoint = os.getenv("TRACELOOP_BASE_URL") - Traceloop.init(app_name="hivemind-server", api_endpoint=otel_endpoint) + if not otel_endpoint: + logging.error("TRACELOOP_BASE_URL is not set.") + return + Traceloop.init(app_name="hivemind-worker", api_endpoint=otel_endpoint) logging.info("Traceloop initialized.") From 3b3814b9aab00c734d0941acac50680549fc9784 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 15:32:41 +0200 Subject: [PATCH 43/50] remove unused import --- worker/celery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/worker/celery.py b/worker/celery.py index 10ffaea..8a4069a 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,6 +1,5 @@ from celery import Celery from utils.credentials import load_rabbitmq_credentials -from utils.traceloop import init_tracing rabbit_creds = load_rabbitmq_credentials() From 9621d44ea6e6aab5f0787e3cbca8ebcd26a21bda Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 16:49:14 +0200 Subject: [PATCH 44/50] isort celery --- .isort.cfg | 2 -- worker/celery.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index b871833..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -check = false diff --git a/worker/celery.py b/worker/celery.py index 8a4069a..8c3fdfb 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,6 +1,6 @@ from celery import Celery -from utils.credentials import load_rabbitmq_credentials +from utils.credentials import load_rabbitmq_credentials rabbit_creds = load_rabbitmq_credentials() user = rabbit_creds["user"] From 00d13697d5f81a5fd593ab3a2514666e0a6e8281 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 16:49:52 +0200 Subject: [PATCH 45/50] isort tasks.py --- worker/tasks.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index e4a98bd..9e284e0 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -3,25 +3,20 @@ import logging from typing import Any -from celery.signals import ( - task_postrun, - task_prerun, -) +from celery.signals import task_postrun, task_prerun from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( - InteractionCallbackData, -) -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ + InteractionCallbackData +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue from subquery import query_multiple_source from utils.data_source_selector import DataSourceSelector -from worker.utils.fire_event import job_send -from worker.celery import app from utils.traceloop import init_tracing +from worker.celery import app +from worker.utils.fire_event import job_send @app.task From 2b7dec76feb44bb35cff9a361ae85dc0f1f3487e Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 16:50:10 +0200 Subject: [PATCH 46/50] black tasks.py --- worker/tasks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 9e284e0..3bf8ddd 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -5,10 +5,12 @@ from celery.signals import task_postrun, task_prerun from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ - InteractionCallbackData -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( + InteractionCallbackData, +) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue From 10e08d32dbd57d744bd96e0398061c4e0cfa054b Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 16:50:39 +0200 Subject: [PATCH 47/50] black tasks.py --- worker/tasks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 3bf8ddd..9e284e0 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -5,12 +5,10 @@ from celery.signals import task_postrun, task_prerun from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( - InteractionCallbackData, -) -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( - ChatInputCommandInteraction, -) +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ + InteractionCallbackData +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ + ChatInputCommandInteraction from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue From 1eb0355a5962f006e52fe809c001f60d26baf416 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 17:01:39 +0200 Subject: [PATCH 48/50] black tasks.py --- worker/tasks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 9e284e0..3bf8ddd 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -5,10 +5,12 @@ from celery.signals import task_postrun, task_prerun from tc_messageBroker.rabbit_mq.event import Event -from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import \ - InteractionCallbackData -from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import \ - ChatInputCommandInteraction +from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( + InteractionCallbackData, +) +from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import ( + ChatInputCommandInteraction, +) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue From d565495d6fb520a4b93edeee617c696aeab45194 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sat, 29 Jun 2024 19:13:34 +0330 Subject: [PATCH 49/50] fix: isort linter issues! --- server.py | 1 - utils/traceloop.py | 5 +++-- worker/celery.py | 1 - worker/tasks.py | 3 +-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server.py b/server.py index 778d3a1..35c9cc2 100644 --- a/server.py +++ b/server.py @@ -10,7 +10,6 @@ ChatInputCommandInteraction, ) from tc_messageBroker.rabbit_mq.queue import Queue - from utils.credentials import load_rabbitmq_credentials from utils.fetch_community_id import fetch_community_id_by_guild_id from worker.tasks import ask_question_auto_search diff --git a/utils/traceloop.py b/utils/traceloop.py index 4b1c1c9..9704719 100644 --- a/utils/traceloop.py +++ b/utils/traceloop.py @@ -1,7 +1,8 @@ +import logging +import os + from dotenv import load_dotenv from traceloop.sdk import Traceloop -import os -import logging load_dotenv() diff --git a/worker/celery.py b/worker/celery.py index 8c3fdfb..bb67f4b 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -1,5 +1,4 @@ from celery import Celery - from utils.credentials import load_rabbitmq_credentials rabbit_creds = load_rabbitmq_credentials() diff --git a/worker/tasks.py b/worker/tasks.py index 3bf8ddd..f833a9d 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -4,6 +4,7 @@ from typing import Any from celery.signals import task_postrun, task_prerun +from subquery import query_multiple_source from tc_messageBroker.rabbit_mq.event import Event from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import ( InteractionCallbackData, @@ -13,8 +14,6 @@ ) from tc_messageBroker.rabbit_mq.payload.payload import Payload from tc_messageBroker.rabbit_mq.queue import Queue - -from subquery import query_multiple_source from utils.data_source_selector import DataSourceSelector from utils.traceloop import init_tracing from worker.celery import app From 50d95e8400b47851927b1326230a24d93ccf5585 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sat, 29 Jun 2024 17:58:32 +0200 Subject: [PATCH 50/50] remove log on finished --- worker/tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index f833a9d..d949b96 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -104,7 +104,6 @@ def ask_question_auto_search( queue_name=Queue.DISCORD_BOT, content=response_payload, ) - logging.info("FINISHED JOB") except Exception as exp: logging.error(f"Exception {exp} | during processing the question {question}") response_payload = Payload.DISCORD_BOT.INTERACTION_RESPONSE.Edit( @@ -118,7 +117,6 @@ def ask_question_auto_search( queue_name=Queue.DISCORD_BOT, content=response_payload, ) - logging.info("FINISHED JOB WITH EXCEPTION") @task_prerun.connect