fix: Optimised writing of fetched item classification codes to

the database.
navariltd · Aug 1, 2024 · 40a447f · 40a447f
1 parent b43c03c
commit 40a447f
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 44 deletions.
diff --git a/kenya_compliance/kenya_compliance/background_tasks/tasks.py b/kenya_compliance/kenya_compliance/background_tasks/tasks.py
@@ -319,48 +319,41 @@ def update_countries(data: dict) -> None:
 
 
 def update_item_classification_codes(response: dict) -> None:
-    # TODO: Optimise this handler
-    # doc: Document | None = None
-    code_lists = response["data"]["itemClsList"]
-
-    # existing_records = {
-    #     doc["itemclscd"]: doc
-    #     for doc in frappe.get_all(
-    #         ITEM_CLASSIFICATIONS_DOCTYPE_NAME, fields=["*"], order_by="creation ASC"
-    #     )
-    # }
-    update_values = []
-
-    for item_classification in code_lists:
-        # doc = existing_records.get(item_classification["itemClsCd"])
-
-        # if doc:
-        update_values.append(
-            [
-                item_classification["itemClsCd"],  # Duplicated for name value
-                item_classification["itemClsCd"],
-                item_classification["itemClsNm"],
-                item_classification["itemClsLvl"],
-                item_classification["taxTyCd"],
-                item_classification["mjrTgYn"],
-                item_classification["useYn"],
-            ]
-        )
-
-        # else:
-        #     doc = frappe.new_doc(ITEM_CLASSIFICATIONS_DOCTYPE_NAME)
-        #     doc.itemclscd = item_classification["itemClsCd"]
-        #     doc.itemclslvl = item_classification["itemClsLvl"]
-        #     doc.itemclsnm = item_classification["itemClsNm"]
-        #     doc.taxtycd = item_classification["taxTyCd"]
-        #     doc.useyn = 1 if item_classification["useYn"] == "Y" else 0
-        #     doc.mjrtgyn = item_classification["mjrTgYn"]
-
-    frappe.db.bulk_insert(
-        ITEM_CLASSIFICATIONS_DOCTYPE_NAME,
-        ["name", "itemclscd", "itemclsnm", "itemclslvl", "taxtycd", "mjrtgyn", "useyn"],
-        update_values,
-        chunk_size=20000,
-    )
+    code_list = response["data"]["itemClsList"]
+    existing_classifications = {
+        cls["name"]: cls
+        for cls in frappe.get_all(ITEM_CLASSIFICATIONS_DOCTYPE_NAME, ["*"])
+    }
+
+    for item_classification in code_list:
+        if item_classification["itemClsCd"] in existing_classifications:
+            update_query = f"""
+                UPDATE `tab{ITEM_CLASSIFICATIONS_DOCTYPE_NAME}`
+                SET itemclscd = '{item_classification["itemClsCd"]}',
+                    itemclslvl = '{item_classification["itemClsLvl"]}',
+                    itemclsnm = '{item_classification["itemClsNm"].replace("'", " ")}',
+                    taxtycd = '{item_classification["taxTyCd"]}',
+                    useyn = '{1 if item_classification["useYn"] == "Y" else 0}',
+                    mjrtgyn  = '{1 if item_classification["mjrTgYn"] == "Y" else 0}'
+                WHERE name = '{item_classification["itemClsCd"]}';
+            """
+
+            frappe.db.sql(update_query)
+
+        else:
+            insert_query = f"""
+                INSERT INTO `tab{ITEM_CLASSIFICATIONS_DOCTYPE_NAME}`
+                    (name, itemclscd, itemclslvl, itemclsnm, taxtycd, useyn, mjrtgyn)
+                VALUES
+                    ('{item_classification["itemClsCd"]}',
+                     '{item_classification["itemClsCd"]}',
+                     '{item_classification["itemClsLvl"]}',
+                     '{item_classification["itemClsNm"].replace("'", " ")}',
+                     '{item_classification["taxTyCd"]}',
+                     '{1 if item_classification["useYn"] == "Y" else 0}',
+                     '{1 if item_classification["mjrTgYn"] == "Y" else 0}');
+            """
+
+            frappe.db.sql(insert_query)
 
     frappe.db.commit()
diff --git a/...ce/doctype/navari_kra_etims_item_classification/navari_kra_etims_item_classification.json b/...ce/doctype/navari_kra_etims_item_classification/navari_kra_etims_item_classification.json
@@ -66,9 +66,10 @@
       "label": "Is Used"
     }
   ],
+  "in_create": 1,
   "index_web_pages_for_search": 1,
   "links": [],
-  "modified": "2024-07-01 11:11:51.711455",
+  "modified": "2024-08-01 17:19:27.314316",
   "modified_by": "Administrator",
   "module": "Kenya Compliance",
   "name": "Navari KRA eTims Item Classification",