From 9ef00cd1327877867db0a893186998fe96e24a60 Mon Sep 17 00:00:00 2001 From: Mo Memarian Date: Wed, 3 Jul 2024 00:27:19 +0000 Subject: [PATCH] #9892: Device only op report --- tt_metal/tools/profiler/process_ops_logs.py | 112 ++++++++++++++++++-- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/tt_metal/tools/profiler/process_ops_logs.py b/tt_metal/tools/profiler/process_ops_logs.py index b9c8802aa17..9c551fe4cb0 100755 --- a/tt_metal/tools/profiler/process_ops_logs.py +++ b/tt_metal/tools/profiler/process_ops_logs.py @@ -73,15 +73,23 @@ ] +def csv_header_format(header): + return header.replace("_", " ").upper() + + def import_tracy_op_logs(): logger.info(f"Importing ops logs") - tracyOpTimesLog = os.path.join(PROFILER_LOGS_DIR, TRACY_OPS_TIMES_FILE_NAME) - tracyOpDataLog = os.path.join(PROFILER_LOGS_DIR, TRACY_OPS_DATA_FILE_NAME) - ops = {} signposts = {} signpostsCount = 0 cached_ops = {} + + tracyOpTimesLog = os.path.join(PROFILER_LOGS_DIR, TRACY_OPS_TIMES_FILE_NAME) + tracyOpDataLog = os.path.join(PROFILER_LOGS_DIR, TRACY_OPS_DATA_FILE_NAME) + + if not os.path.isfile(tracyOpTimesLog) or not os.path.isfile(tracyOpDataLog): + return ops, signposts + with open(tracyOpDataLog, "r", newline="") as csvFile: opDataDicts = csv.DictReader(csvFile, delimiter=";", quotechar="`") opsData = [] @@ -201,6 +209,93 @@ def append_device_data(ops, deviceLogFolder): return deviceOps +def get_device_data_generate_report(deviceLogFolder, outputFolder, date, nameAppend): + deviceTimesLog = os.path.join(deviceLogFolder, PROFILER_DEVICE_SIDE_LOG) + devicePreOpTime = {} + deviceOps = {} + i = 0 + rowDicts = [] + + outFolder = PROFILER_OUTPUT_DIR + if outputFolder: + outFolder = outputFolder + + name = OUT_NAME + outFolder = os.path.abspath(outFolder) + + if nameAppend: + name += f"_{nameAppend}" + outFolder = os.path.join(outFolder, nameAppend) + + if date: + dateStr = f"{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" + name += f"_{dateStr}" + outFolder = os.path.join(outFolder, dateStr) + + allOpsCSVPath = os.path.join(outFolder, f"{name}.csv") + + if os.path.isfile(deviceTimesLog): + logger.info(f"Getting device only ops data") + setup = device_post_proc_config.default_setup() + setup.deviceInputLog = deviceTimesLog + deviceData = import_log_run_stats(setup) + logger.info(f"Generating device op report ...") + freq = deviceData["deviceInfo"]["freq"] + for device in deviceData["devices"]: + deviceOps[device] = [] + deviceOpsTime = deviceData["devices"][device]["cores"]["DEVICE"]["riscs"]["TENSIX"]["ops"] + for deviceOpTime in deviceOpsTime: + i += 1 + rowDict = {csv_header_format("global_call_count"): i} + deviceOp = {} + cores = set() + for timeID, ts, statData, risc, core in deviceOpTime["timeseries"]: + if "zone_name" in timeID.keys() and "FW" in timeID["zone_name"]: + if core not in cores: + cores.add(core) + deviceOp["core_usage"] = {"count": len(cores), "cores": [str(core) for core in cores]} + deviceOp["device_time"] = { + analysis: data["series"] for analysis, data in deviceOpTime["analysis"].items() + } + deviceOp["global_call_count"] = i + for analysis, data in deviceOp["device_time"].items(): + for sample in data: + sample["duration_ns"] = sample["duration_cycles"] * 1000 / freq + deviceOps[device].append(deviceOp) + + for analysis, analysisData in deviceOp["device_time"].items(): + headerField = f"{csv_header_format(analysis)} [ns]" + assert len(analysisData) == 1, "Unexpected device data format" + rowDict[headerField] = f"{analysisData[0]['duration_ns']:.0f}" + if analysis == "device_fw_duration": + rowDict["DEVICE FW START CYCLE"] = analysisData[0]["start_cycle"] + rowDict["DEVICE FW END CYCLE"] = analysisData[0]["end_cycle"] + if device in devicePreOpTime.keys(): + rowDict["OP TO OP LATENCY [ns]"] = round( + 1000 * (analysisData[0]["start_cycle"] - devicePreOpTime[device]) / freq + ) + else: + rowDict["OP TO OP LATENCY [ns]"] = 0 + devicePreOpTime[device] = analysisData[0]["end_cycle"] + rowDicts.append(rowDict) + + with open(allOpsCSVPath, "w") as allOpsCSV: + allHeaders = [] + for header in OPS_CSV_HEADER: + if header in rowDicts[-1].keys(): + allHeaders.append(header) + writer = csv.DictWriter(allOpsCSV, fieldnames=allHeaders) + writer.writeheader() + for rowDict in rowDicts: + for field, fieldData in rowDict.items(): + rowDict[field] = str(fieldData).replace(",", ";") + writer.writerow(rowDict) + logger.info(f"Device only OPs csv generated at: {allOpsCSVPath}") + else: + logger.info("No device logs found") + return deviceOps + + def generate_reports(ops, deviceOps, signposts, outputFolder, date, nameAppend): logger.info(f"OPs' perf analysis is finished! Generating reports ...") outFolder = PROFILER_OUTPUT_DIR @@ -218,6 +313,7 @@ def generate_reports(ops, deviceOps, signposts, outputFolder, date, nameAppend): dateStr = f"{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" name += f"_{dateStr}" outFolder = os.path.join(outFolder, dateStr) + allOpsCSVPath = os.path.join(outFolder, f"{name}.csv") logger.info(f"Copying runtime artifacts") os.system(f"rm -rf {outFolder}; mkdir -p {outFolder}") @@ -301,9 +397,6 @@ def add_io_data(tensors, ioType): if count > tensorCSVData[ioType]["maxCount"]: tensorCSVData[ioType]["maxCount"] = count - def csv_header_format(header): - return header.replace("_", " ").upper() - def row_compare(row): ret = 0 if type(row) is str and "sp" in row: @@ -425,9 +518,12 @@ def row_compare(row): def process_ops(output_folder, name_append, date): ops, signposts = import_tracy_op_logs() - deviceOps = append_device_data(ops, PROFILER_LOGS_DIR) + if ops: + deviceOps = append_device_data(ops, PROFILER_LOGS_DIR) + generate_reports(ops, deviceOps, signposts, output_folder, date, name_append) - generate_reports(ops, deviceOps, signposts, output_folder, date, name_append) + else: + deviceOps = get_device_data_generate_report(PROFILER_LOGS_DIR, output_folder, date, name_append) @click.command()