Update utils for statvar processor (#1132)

* update utils * lint fix * add chardet requirement * fix review comments * fix review comments * fix review comments * add process time, mem counters * lint * add psutil requirement
datacommonsorg · Jan 2, 2025 · b4ebd73 · b4ebd73
1 parent c1d3abe
commit b4ebd73
Show file tree

Hide file tree

Showing 9 changed files with 769 additions and 471 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -2,6 +2,7 @@
 
 absl-py
 arcgis2geojson
+chardet
 dataclasses==0.6
 datacommons==1.4.3
 frozendict
@@ -17,6 +18,7 @@ lxml==4.9.1
 numpy==1.26.4
 openpyxl>=3.1.0
 pandas
+psutil
 pylint
 pytest
 requests==2.27.1

diff --git a/requirements_all.txt b/requirements_all.txt
@@ -3,6 +3,7 @@
 absl-py
 arcgis2geojson
 chembl-webresource-client>=0.10.2
+chardet
 dataclasses==0.6
 datacommons==1.4.3
 deepdiff==6.3.0
@@ -25,6 +26,7 @@ netCDF4==1.6.4
 numpy==1.26.4
 openpyxl>=3.1.0
 pandas
+psutil
 pylint
 pytest
 rasterio