From e52ac934199e5440fe595908f433bf77a314e15d Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Fri, 6 Dec 2024 16:59:49 +0100 Subject: [PATCH] various bug fixes --- app.py | 7 +- data/data_template.xlsx | Bin 9609 -> 9683 bytes main.py | 283 ++++++++++++++++++++++------------------ utils.py | 103 ++++++++------- 4 files changed, 221 insertions(+), 172 deletions(-) diff --git a/app.py b/app.py index b9225fc..42a921e 100644 --- a/app.py +++ b/app.py @@ -191,7 +191,7 @@ def create_distrib(): return redirect(url_for("name_distrib")) if ( datetime.strptime(request.form["distrib_date"], "%Y-%m-%d").date() - > datetime.now().date() + < datetime.now().date() ): flash("Date of the distribution cannot be in the past.") return redirect(url_for("name_distrib")) @@ -240,9 +240,8 @@ def get_list_distrib(): distrib_features["Date"] = distrib_.date distrib_features["Items distributed"] = distrib_.items distrib_features["Donor"] = distrib_.donor - distrib_list.loc[len(distrib_list), distrib_features.keys()] = ( - distrib_features.values() - ) + df_dictionary = pd.DataFrame([distrib_features]) + distrib_list = pd.concat([distrib_list, df_dictionary], ignore_index=True) return distrib_list diff --git a/data/data_template.xlsx b/data/data_template.xlsx index 82273029565ed715158b10feca0ad49bfb07db5f..bba505fc979e87c399a2a2f18534a744163aa7f0 100644 GIT binary patch delta 3159 zcmZ8kc{J4P8y~wN3<+7vGE0adk)_BU;)bk|r7)H;c9ZSP*dmg~PIi+eLzYC1xQgsc zm<+OHO?EMs_;v67o!dG0`RhIB^ZfUIwkOWulR-6w5s-!liVe_$K)Q?|5C;eZ!g?SB zF+T2&7>v6#7VS}NW{aW7aEIpAo!IYRQv^eV-7@mS%~v%`Cn+6W&={;@fU^q2?bUbd z$|7|_NLzuHxX>Bugm1q*Z{BY7b&i_?#QV6&$4Q*{vaZ(?Q0ih*^p*D?EhXe6QtpFQ zWg-E!M76NjpT$YG;`Q1fVMCm+dv^u|zwjcTP^1qRsWETL=OF}X-Yh%Yd$TV?iXI%Q zCdx3+b%pt57+Z{W)q^f)iAokYnyxT2pJypMC!L@zIYe$ba~o4JWMF1Y`#zr4lSBT3 zjyAs?x_HPBCcGLtpc~A#&a4{2q4{}@z5uALWXW}N8m0t*vMv|*Z*hch(IcUs^V~O- zi_Z;yk9fnMYq`9X_@eN%`3^k~bzO(G#k$_Y5wf0$`gytml7t+!sUB}{dYCpJX#H9r zoIo!hkn7!DL(gGf9`OAGj@ae;fkCUSwR+{=CDpM9We&9UdCc8)#c8$@uB~pm-`0V8 zv&7I$Ng3HD=Xa9CvpejT%tH~$1XW&awkCpCHv4~{L&KU#NG2Bw<>V%X-a(CCB70j5%CW=bRV^Jy~|Nf+eHR~p15H9xY~g$O&GC= z^A=ZwXE*q?XHcpcOy^qINnd*Mg4>AeD_eKLNG9jzJKLO)2EfDpySq1I4goRND)$X2~aPizAG1&YV zXm?YcJ6mut!Ql8qsI$q!bHeJB#)y0UTfXJbs|8gx`9!l#}pH zI}pBrj=Q!WEJq908+m4|K?efi@vNuBfNU8^%(~tJ8jE~s5(qOa%-~vzBHnKhU*5x1bke~_ zH;J>AYx%{9>K*U>-QL;!-cbe6W8s2Y7&>Q#wfzO^y0^gcIc1x&dgF);r%tm54{28KcdX9iH_E! zu5st5Trj?hQASgnJ~SoiwKFL3zcmdZ6y2nT6%`lDP`**LZ#ZlAHO%*q<#>CGOpXG) z@Xm|~(hd_=yQIoMyc79ihb0?m=2Cdmauks1h^Tma{J5V?kDiUCDkr!DOFqVWR~_um z$lgldu{8p~_AdyVCq92gN=`T;<+kfsFiY=$t zL3SlpdSX`J`lIc>IwG`^g&wEYnJzylf2Km;7dx(g=IJSYu0xpg9o&LeEvzeN;0|v7 zy}yWl!svSbOE9>w0&O4m%J9&4I@6h!k1%%bb;%f~W$Lz;RS49~`Ojk+aQ#X#dkeJp zf~5;S+8(dW_D4%}uk=-EPj8V;;0RPpWY=N>L(9Aw=vo1NNPHA^88Q^}z9L((GFnZvLueUt@@yrfE_2kFPvh<7 z-5;cm#V^4gSHi|`M66|&HJXFjr9%=jy8K&5;H`PmHR zT)kU8E_IKGCaV1O+xl155R=6_aM#cRNrHXJu#O3UhHq{H(Teu9c`Dd^NVkxB%gIr^FvVJg?eCl0|A|LE6?D-On3+HIckiWV4=P=*6sM z3+Ay)Jtkagpch&r4ZRM#zIQtu)&foRm8w(cv1Uqjmx|XPePCnUN{&%!c&abG`%K9% z=G8bUZ|+vG3-qyW2RxD7w=#Gak$jl3YpQvX*D*!U<;Fc!qmzzPbv$seIoV9x#WFmG zW;1XhU5QzXl;LN15};{cVMz1bp>zh8QXyCJ(!M0x=G2tOB>5Xl7@1;y>n1;Mj%Qur zpCxKq31Tzctr2?Flep>Fpij?6LL1{E*9ZF#UI|*WxY$N4v~0b=r1t>b^EOI{O*myA z`1{I=#q_NKiRa%5T5bSB&UK0x98=M=?8S$ZK5!tXdCpSgbyli%tBq&?_D5M;;U_kJ zpd+JX+CxYp$}Eng(qsgi^ca)aQ3McF9lg=UV%eM#&-hoA4JKy)*sHPrjDf-vJ86r3 zv`wh_sg(Z3JweUD+MWGpj!Q<1YV+d(r-CkX{V8@54Xp_Ard6Eg z0`(Ls#dihD*Q{N0oe%iTP-j;CbS;XcauH5dJZ@oW#&1WG5n)wm*5v80)OK4~>+otT zf3RuWek_u5Hg|fR+Lr(fJ!81$Lf##hX{rS|?Dcy}NhG;%y^=ziYe ziMou{FzsDci%*F0=h)yn820k=QiB97<=)!1*SB|%Xs5cRf3{@bT<*y>M2aCF;k@LZ zU92j)$nTwWm%6(TWp1(M*3oRZ?arGBce7a zDgvP0tZi;K(MkrKkV7r$d<}O*P&O@Rm8onpl2xjTr?>f>)$f4h25BLa5^mxuGYQ&# zI(ae+BsLfD>K)u0yZkd&m4$sy*W*f4sks0jf5W|$KLIRbj&KT%A<}6)(AKOTZ5Pks zypxSwE&}_;52hV`wx~(1Q5x|FDA}X7g9gn*xvH>BmK>CI@`wqz_XuG!r?!OrCoC=b zJ82n={cG$@YvDpr8X)AUu|b;*ck~p)iDqZ5ZI)3x4~6~qt}r{jWIRTsaU~wj^|Xh0 z@}+LlEmm|6rnDHAAT(4MAGc-SM#9X_X@QgdU_y&0T-q5832s{~9$TgtMo zk66V;6t`f^O&qef0V`}kUB$`b6qV=V?P==;&H)QTLyuIP0Jnvs5pGMl<4%?qZ*D0s zw#VQFzP0dzb8$;i+#y=uBFlopr}-pG)s-K0CQP5NqB9*=V+wMbePKX$n_A`$?BWT5 z=bw3Jj__}?OqpB6Qu z>BJ+&qy+yhDNG;`-!DS`o+DN~& delta 3081 zcmZ9Oc{J4PAIE2`4TB;3*v$|kTT)~fGm<4Djgkg47&}q;GD$au$uJ>1WyzM&wbbN} zC2QHT@0XaeCQA$E=l1)Z+d04IujicS^ZfHX@7L?R2d(Z~RgZH3BlyDH4i*qdp92Kq z1A#!{z9(+_2l=@B`}?Sc6MTyt2>wshVFw0yCiuJF3+p4+wq)fKrq6c@N~Xq>(^xNT zu;@^&uR5$YtwT}cYS$WtW9K3}=a<%UUUffry&Mj&OtcO919wj*x!b|Rq%xkdl-#;5 z-|MWPb?I2=JzOuq(O}%oZP4tQO5>iXjF%XjTbO1?p`z)lB zws>6Y^wOWl1B@)=tP~L5{OsB1)s#OS7Stvcr+`7#^GQkgao<`XMT}SPB=WaDqicIAr_+jlbgN-@|KXf7gz6lh4=C9!MEE}~77 zH_yMCj6QN*KmP$_+XnD6>4}? zQ*S+!Nz$}(RnWjzC06eroFcPV)+{ZydQjt*5SLR9j+0BA6Sc8bWdZwG0tN;qJAczC!X<0Le0(0(@!><#fd5r z#UsD>&|b?o*5&ICc2K6y4`f=FNmHPA48G7az2;S0I{U?6NoAm6XR>>rr2T$=R(hYE zoX;vq9f4@EK;9Z~Ny)QCH5*zHAc~rfib_b{xd~G(WP$^`RpFAm-vm7yWhN>dVp==aLGHh z?h%5Dam7AE-h>*>!Pu`HhR)ebp0@*=O^f?1hH(;KX<=PYh;_5Ts8-e}Qkul$LR zAAMi= zIOQL*DogAhRD5}^v_RO}Pk)(axuY7M1)OFk}=-Cxtr zRha+6(13Gx0doCb4_gl;G8D}p6`NC-S}Eo~0Wb)}WFGpA$xKbL;%%0J8C99+#-tLP z;!|)9M~nrWh6L@mft?G9#-fuU7K>+79(R@mY#WHfD=rPL8b8YkLN`w(fv;v4Ep$E& zo%-kz84&&OEJ26p{ZV+kwmw%{oliYGv~aXsDEQhu2L4(*ejI1j@*&=mXKm;E)JkkKkV#cAW5=P=`iIR|BxEwXU%F;}BmF>x-=Hjn|Fz+^P~( z6-`1luup@}tjt(^PpQY%LC37*>^}6%@}b(D5(OC<8JRw5v$M_Zf0SZGFa&|URgJHn zqC=E2B3Feoaku=f6Y!~>FOaZHJnJh)JtE_fJzMc|h)6m!qlsGPcGX~4e5#Y;GMdR# zs?vh9du3%$>f0?gv#$)eS55lf6l3X~6PEVa=dxY*2qxf_{|oh$?|=Ih*5>K(?Z{8HWD^O#9p!gIh>5+_Wra6g^oM2>dRkNiR1y zu-WZALXm{}V3pDZ_n8b+FWk`DKH}b_o_wJG86EuVu&Y(mw?rD61`mb;9&>9X6loP{ zSo&qyiDeZ;?*sK!F*U@`D>?Cno- zuJX;*$0y%?pnf)_P)JB^k9T3)njwcvXPZzJ+#~rnQX6ef1#X=EftsP1TFn)y^hr49 zY^@R_3@HZ3ND5vi`t@#a9^cb%sPRS@g=vy^DlOG}0DdR776G$Qk6ry56T62*D_W%J z@m}c zw4A;JX1g)GIeFPA_0OHmnqE+=K6>SOt&~6so zx9B^v1`-5_6yeN|3T30)JyB8@_Lq-s&{hWY_+EFWzJEQorMX#i?1qn z?s||1_F_m7-p}h7M!qCtt5$W`-E|^Uc419nl@jJOuyQ?+kEpoK85icu+FA4jWyYlO z8|QC-d(-|TLO=A!MiR{w0k2*fx)8*n8`uB$hp!`#yQgBc&>R-?EGE5%vENWL)5i*0 z0km-GQZTxk2Gj1D)GlgLS{vWxQ3cmKcV+T{?@p#c3|)3jBHxYOZ)E3;USpPgz?kK@tH5 z9;Vn?wWtf;ondF1c2WtMeozR3Q~5TGlPJ_yN16<6(}AEoM*#nEzn7-ckv(UZqO*cg zjRrcQ8|-f{p*{aF)8H}+LMmVOWV{@Wt2cQ=7!O-2mY;DX!Ot=hW zx9J~PjRK%2wPmu~5OUwzm;ch<>Ah>5ar#MD==jYuZP`?I>9)?O+99m(gpRj}DKs~C zc6W8K+xBER7Es|mX2V`Q2@397hytc0mj!#{_rX4%Q^Pa2oMP*jH&wMh($6=1RtU#p z@M(X~NcBp3KMa7y@||lJ_xeZ(WZL@L=Cq;&Qgjh0<4f8Ksgu!L{AxQt^XBtTvw!Cc zyi7`0qokI5wra^p2VIL!+Jpb96PwZ}SRwz`XR@;V1u&f)A+HMmw_+TcT=X!Bzt 0 and 'recipient' in data.columns: - number_recipients = len(data[data['recipient'] == 'Yes']) - return render_template('index.html', - distrib_name=str(session['distrib_name']), - distrib_place=str(session['distrib_place']), - distrib_date=str(session['distrib_date']), - number_beneficiaries=number_beneficiaries, - number_recipients=number_recipients) + if number_beneficiaries > 0 and "recipient" in data.columns: + number_recipients = len(data[data["recipient"] == "Yes"]) + return render_template( + "index.html", + distrib_name=str(session["distrib_name"]), + distrib_place=str(session["distrib_place"]), + distrib_date=str(session["distrib_date"]), + number_beneficiaries=number_beneficiaries, + number_recipients=number_recipients, + ) -@main.route('/profile') +@main.route("/profile") def profile(): - return render_template('profile.html', name=current_user.name) \ No newline at end of file + return render_template("profile.html", name=current_user.name) diff --git a/utils.py b/utils.py index 8ad99aa..68ae6bb 100644 --- a/utils.py +++ b/utils.py @@ -5,37 +5,44 @@ import os import azure.cosmos.cosmos_client as cosmos_client from dotenv import load_dotenv + load_dotenv() def get_cosmos_db(): settings = { - 'host': os.environ.get('ACCOUNT_HOST', 'https://emergencycosmos.documents.azure.com:443/'), - 'master_key': os.getenv('COSMOS_KEY'), - 'database_id': os.environ.get('COSMOS_DATABASE', 'ReliefApp'), - 'container_id': os.environ.get('COSMOS_CONTAINER', 'Beneficiaries'), + "host": os.environ.get( + "ACCOUNT_HOST", "https://emergencycosmos.documents.azure.com:443/" + ), + "master_key": os.getenv("COSMOS_KEY"), + "database_id": os.environ.get("COSMOS_DATABASE", "ReliefApp"), + "container_id": os.environ.get("COSMOS_CONTAINER", "Beneficiaries"), } - client = cosmos_client.CosmosClient(settings['host'], - {'masterKey': settings['master_key']}, - user_agent="ReliefApp", - user_agent_overwrite=True) - return client.get_database_client(settings['database_id']) + client = cosmos_client.CosmosClient( + settings["host"], + {"masterKey": settings["master_key"]}, + user_agent="ReliefApp", + user_agent_overwrite=True, + ) + return client.get_database_client(settings["database_id"]) + cosmos_db = get_cosmos_db() + def get_local_data_path(user_email, distrib_id): - data_dir = 'instance' + data_dir = "instance" os.makedirs(data_dir, exist_ok=True) return f"{data_dir}/user_{user_email}_distrib_{distrib_id}_data.csv" def delete_beneficiary_data(user_email, distrib_id): if os.getenv("MODE") == "online": - cosmos_container = cosmos_db.get_container_client('Beneficiaries') + cosmos_container = cosmos_db.get_container_client("Beneficiaries") item_list = query_items_by_partition_key(cosmos_container, user_email) item_list = filter_by_distribution(item_list, distrib_id) for item in item_list: - cosmos_container.delete_item(item=item['id'], partition_key=user_email) + cosmos_container.delete_item(item=item["id"], partition_key=user_email) elif os.getenv("MODE") == "offline": database = get_local_data_path(user_email, distrib_id) if os.path.exists(database): @@ -46,31 +53,32 @@ def get_beneficiary_data(user_email, distrib_id): df = pd.DataFrame() if os.getenv("MODE") == "online": - cosmos_container = cosmos_db.get_container_client('Beneficiaries') + cosmos_container = cosmos_db.get_container_client("Beneficiaries") item_list = query_items_by_partition_key(cosmos_container, user_email) item_list = filter_by_distribution(item_list, distrib_id) df = pd.DataFrame(item_list) elif os.getenv("MODE") == "offline": database = get_local_data_path(user_email, distrib_id) if os.path.exists(database): - df = pd.read_csv(database, index_col=0, sep=';') + df = pd.read_csv(database, index_col=0, sep=";") else: return None if df.empty: return None else: - df = df.drop(columns=['partitionKey', 'id', 'distrib_id'], errors='ignore') - df = df.drop(columns=[c for c in df.columns if c.startswith('_')]) + df = df.drop(columns=["partitionKey", "id", "distrib_id"], errors="ignore") + df = df.drop(columns=[c for c in df.columns if c.startswith("_")]) return df def get_beneficiary_entry(beneficiary_id, user_email, distrib_id): if os.getenv("MODE") == "online": - cosmos_container = cosmos_db.get_container_client('Beneficiaries') + cosmos_container = cosmos_db.get_container_client("Beneficiaries") try: - data = cosmos_container.read_item(item=beneficiary_id, - partition_key=user_email) - for internal_field in [k for k in data.keys() if k.startswith('_')]: + data = cosmos_container.read_item( + item=beneficiary_id, partition_key=user_email + ) + for internal_field in [k for k in data.keys() if k.startswith("_")]: data.pop(internal_field) return data except exceptions.CosmosResourceNotFoundError: @@ -78,7 +86,7 @@ def get_beneficiary_entry(beneficiary_id, user_email, distrib_id): elif os.getenv("MODE") == "offline": database = get_local_data_path(user_email, distrib_id) if os.path.exists(database): - df = pd.read_csv(database, sep=';', dtype={'id': str}).set_index('id') + df = pd.read_csv(database, sep=";", dtype={"id": str}).set_index("id") if beneficiary_id in df.index: return df.loc[beneficiary_id].to_dict() else: @@ -90,8 +98,10 @@ def get_beneficiary_entry(beneficiary_id, user_email, distrib_id): def update_beneficiary_entry(beneficiary_id, user_email, distrib_id, replace_body): if os.getenv("MODE") == "online": try: - cosmos_container = cosmos_db.get_container_client('Beneficiaries') - read_item = cosmos_container.read_item(item=beneficiary_id, partition_key=user_email) + cosmos_container = cosmos_db.get_container_client("Beneficiaries") + read_item = cosmos_container.read_item( + item=beneficiary_id, partition_key=user_email + ) for key in replace_body.keys(): read_item[key] = replace_body[key] cosmos_container.replace_item(item=read_item, body=read_item) @@ -101,11 +111,11 @@ def update_beneficiary_entry(beneficiary_id, user_email, distrib_id, replace_bod elif os.getenv("MODE") == "offline": database = get_local_data_path(user_email, distrib_id) if os.path.exists(database): - df = pd.read_csv(database, sep=';', dtype={'id': str}).set_index('id') + df = pd.read_csv(database, sep=";", dtype={"id": str}).set_index("id") if beneficiary_id in df.index: for key in replace_body.keys(): df.at[beneficiary_id, key] = replace_body[key] - df.to_csv(database, sep=';') + df.to_csv(database, sep=";") return "success" else: return "not_found" @@ -117,45 +127,45 @@ def save_beneficiary_data(data, distrib_id, user_email): data_to_save = [] for ix, row in data.iterrows(): body = { - 'id': str(distrib_id) + str(row['code']), - 'partitionKey': user_email, - 'distrib_id': str(distrib_id) + "id": str(distrib_id) + str(row["code"]), + "partitionKey": user_email, + "distrib_id": str(distrib_id), } for key in row.keys(): - if key != 'id' and key != 'partitionKey' and key != 'distrib_id': + if key != "id" and key != "partitionKey" and key != "distrib_id": body[key] = str(row[key]) data_to_save.append(body) print(data_to_save) if os.getenv("MODE") == "online": - cosmos_container = cosmos_db.get_container_client('Beneficiaries') + cosmos_container = cosmos_db.get_container_client("Beneficiaries") # save to cosmos db for entry in data_to_save: cosmos_container.create_item(body=entry) elif os.getenv("MODE") == "offline": database = get_local_data_path(user_email, distrib_id) df = pd.DataFrame.from_records(data_to_save) - df.index = df['id'] - df = df.drop(columns=['id']) - df.to_csv(database, sep=';') + df.index = df["id"] + df = df.drop(columns=["id"]) + df.to_csv(database, sep=";") def query_items_by_partition_key(container, key): # Including the partition key value of account_number in the WHERE filter results in a more efficient query - items = list(container.query_items( - query="SELECT * FROM r WHERE r.partitionKey=@key", - parameters=[ - {"name": "@key", "value": key} - ] - )) + items = list( + container.query_items( + query="SELECT * FROM r WHERE r.partitionKey=@key", + parameters=[{"name": "@key", "value": key}], + ) + ) return items def filter_by_distribution(item_list, distrib_id): for item in item_list[:]: - if 'distrib_id' in item.keys(): - if str(item['distrib_id']) != str(distrib_id): + if "distrib_id" in item.keys(): + if str(item["distrib_id"]) != str(distrib_id): item_list.remove(item) return item_list @@ -165,8 +175,13 @@ def pandas_to_html(df, replace_values={}, replace_columns={}, titlecase=False): df = df.rename(columns=replace_columns) if titlecase: df.columns = [x.title() for x in df.columns] - columns = df.columns + columns = list(df.columns) rows = [] + print(df) for ix, row in df.iterrows(): - rows.append(row.to_dict()) - return columns, rows \ No newline at end of file + row_dict = row.to_dict() + for key in row.keys(): + row[key] = str(row[key]).strip() + rows.append(row_dict) + print(columns, rows) + return columns, rows