Skip to content

Commit

Permalink
More debugging...
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Oct 20, 2024
1 parent 27c2944 commit 5d1d59f
Showing 1 changed file with 10 additions and 8 deletions.
18 changes: 10 additions & 8 deletions src/pypi_data/combine_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ def append_buffer(


def log_system_stats(path: Path):
disk_usage = psutil.disk_usage(str(path))
cpu_usage = psutil.cpu_percent(percpu=True)
process_mem_percent = psutil.Process().memory_percent()
psutil.disk_partitions(all=False)
mem = psutil.virtual_memory()
log.info(
f"System: cpu={cpu_usage} "
Expand All @@ -56,13 +56,15 @@ def log_system_stats(path: Path):
f"mem_used={ByteSize(mem.used).human_readable(decimal=True)} "
f"mem_percent={mem.percent:.1f}%"
)
log.info(
f"Disk: "
f"total={ByteSize(disk_usage.total).human_readable(decimal=True)} "
f"used={ByteSize(disk_usage.used).human_readable(decimal=True)} "
f"free={ByteSize(disk_usage.free).human_readable(decimal=True)} "
f"percent={disk_usage.percent:.1f}%"
)
for partition in psutil.disk_partitions(all=True):
disk_usage = psutil.disk_usage(partition.mountpoint)
log.info(
f"Disk: ({partition.fstype} - {partition.mountpoint}) "
f"total={ByteSize(disk_usage.total).human_readable(decimal=True)} "
f"used={ByteSize(disk_usage.used).human_readable(decimal=True)} "
f"free={ByteSize(disk_usage.free).human_readable(decimal=True)} "
f"percent={disk_usage.percent:.1f}%"
)


def buffer_mem_size(buffer: Deque[tuple[tuple[int, str], RecordBatch]]) -> int:
Expand Down

0 comments on commit 5d1d59f

Please sign in to comment.