Support extended components

.github/workflows/update_components.yaml

@@ -51,9 +51,10 @@ jobs:
 
           jlcparts fetchtable parts.csv
 
-          jlcparts getlibrary --age 10000 \
+          jlcparts getlibrary --age 0 \
                               --limit 15000 \
                               parts.csv cache.sqlite3
+          jlcparts updatepreferred cache.sqlite3
           jlcparts buildtables --jobs 0 \
                                --ignoreoldstock 120 \
                                cache.sqlite3 web/build/data

jlcparts/datatables.py

@@ -193,8 +193,13 @@ def pullExtraAttributes(component):
     a dictionary
     """
     status = "Discontinued" if component["extra"] == {} else "Active"
+    type = "Extended"
+    if component["basic"]:
+        type = "Basic"
+    if component["preferred"]:
+        type = "Preferred"
     return {
-        "Basic/Extended": "Basic" if component["basic"] else "Extended",
+        "Basic/Extended": type,
         "Package": component["package"],
         "Status": status
     }

jlcparts/lcsc.py

@@ -1,3 +1,4 @@
+import json
 import requests
 import os
 import time
@@ -28,6 +29,41 @@ def makeLcscRequest(url, payload=None):
 
     return requests.get(url, params=newPayload)
 
+def pullPreferredComponents():
+    resp = requests.get("https://jlcpcb.com/api/overseas-pcb-order/v1/getAll")
+    token = resp.cookies.get_dict()["XSRF-TOKEN"]
+
+    headers = {
+        "Content-Type": "application/json",
+        "X-XSRF-TOKEN": token,
+    }
+    PAGE_SIZE = 1000
+
+    currentPage = 1
+    components = set()
+    while True:
+        body = {
+            "currentPage": currentPage,
+            "pageSize": PAGE_SIZE,
+            "preferredComponentFlag": True
+        }
+
+        resp = requests.post(
+            "https://jlcpcb.com/api/overseas-pcb-order/v1/shoppingCart/smtGood/selectSmtComponentList",
+            headers=headers,
+            json=body
+        )
+
+        body = resp.json()
+        for c in [x["componentCode"] for x in body["data"]["componentPageInfo"]["list"]]:
+            components.add(c)
+
+        if not body["data"]["componentPageInfo"]["hasNextPage"]:
+            break
+        currentPage += 1
+
+    return components
+
 if __name__ == "__main__":
     r = makeLcscRequest("https://ips.lcsc.com/rest/wmsc2agent/product/info/C7063")
     print(r.json())

jlcparts/partLib.py

@@ -35,6 +35,8 @@ def dbToComp(comp):
     comp["lcsc"] = lcscFromDb(comp["lcsc"])
     comp["price"] = json.loads(comp["price"])
     comp["extra"] = json.loads(comp["extra"])
+    comp["basic"] = bool(comp["basic"])
+    comp["preferred"] = bool(comp["preferred"])
     return comp
 
 class PartLibraryDb:
@@ -54,6 +56,7 @@ def __init__(self, filepath=None):
                 joints INTEGER NOT NULL,
                 manufacturer_id INTEGER NOT NULL,
                 basic INTEGER NOT NULL,
+                preferred INTEGER NOT NULL DEFAULT 0,
                 description TEXT NOT NULL,
                 datasheet TEXT NOT NULL,
                 stock INTEGER NOT NULL,
@@ -64,11 +67,22 @@ def __init__(self, filepath=None):
             )""")
 
         # Perform migration if we miss last on stock
-        columns = self.conn.execute("pragma table_info(components)")
+        migrated = False
+        columns = list(self.conn.execute("pragma table_info(components)"))
         if "last_on_stock" not in [x[1] for x in columns]:
             self.conn.execute("""
                 ALTER TABLE components ADD COLUMN last_on_stock INTEGER NOT NULL DEFAULT 0;
             """)
+            migrated = True
+
+        # Perform migration if we miss the preferred flag
+        if "preferred" not in [x[1] for x in columns]:
+            self.conn.execute("""
+                ALTER TABLE components ADD COLUMN preferred INTEGER NOT NULL DEFAULT 0;
+            """)
+            migrated = True
+
+        if migrated:
             self.conn.execute("DROP VIEW v_components")
 
         self.conn.execute("""
@@ -104,6 +118,7 @@ def __init__(self, filepath=None):
                     c.joints AS joints,
                     m.name AS manufacturer,
                     c.basic AS basic,
+                    c.preferred as preferred,
                     c.description AS description,
                     c.datasheet AS datasheet,
                     c.stock AS stock,
@@ -292,6 +307,12 @@ def updateJlcPart(self, component, flag=None):
             """, data)
         self._commit()
 
+    def setPreferred(self, lcscSet):
+        cursor = self.conn.cursor()
+        cursor.execute("UPDATE components SET preferred = 0")
+        cursor.execute(f"UPDATE components SET preferred = 1 WHERE lcsc IN ({','.join(len(lcscSet) * ['?'])})",
+                       [lcscToDb(x) for x in lcscSet])
+        self._commit()
 
     def categories(self):
         res = {}

jlcparts/ui.py

@@ -4,6 +4,7 @@
 import click
 
 from jlcparts.datatables import buildtables, normalizeAttribute
+from jlcparts.lcsc import pullPreferredComponents
 from jlcparts.partLib import (PartLibrary, PartLibraryDb, getLcscExtraNew,
                               loadJlcTable, loadJlcTableLazy)
 
@@ -62,6 +63,18 @@ def getLibrary(source, db, age, limit):
     # db.vacuum()
 
 
+
+@click.command()
+@click.argument("db", type=click.Path(dir_okay=False, writable=True))
+def updatePreferred(db):
+    """
+    Download list of preferred components from JLC PCB and mark them into the DB.
+    """
+    preferred = pullPreferredComponents()
+    lib = PartLibraryDb(db)
+    lib.setPreferred(preferred)
+
+
 @click.command()
 @click.argument("libraryFilename")
 def listcategories(libraryfilename):
@@ -152,6 +165,7 @@ def cli():
 cli.add_command(listcategories)
 cli.add_command(listattributes)
 cli.add_command(buildtables)
+cli.add_command(updatePreferred)
 cli.add_command(fetchDetails)
 cli.add_command(fetchTable)
 cli.add_command(testComponent)