From 31e44eb287b2bcef67a322e81d21cf9984a06139 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Apr 2025 05:08:22 +0530 Subject: [PATCH 01/55] Cleanups for openai-call --- script/openai-call/customize.py | 65 +++++++++++++++++++++++++++++ script/openai-call/meta.yaml | 35 ++++++++++++++++ script/openai-call/openai_call.py | 69 +++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 script/openai-call/customize.py create mode 100644 script/openai-call/meta.yaml create mode 100644 script/openai-call/openai_call.py diff --git a/script/openai-call/customize.py b/script/openai-call/customize.py new file mode 100644 index 000000000..9bab431fb --- /dev/null +++ b/script/openai-call/customize.py @@ -0,0 +1,65 @@ +from mlc import utils +import os +import subprocess +import json + + +import yaml + +def write_openai_yaml(model, system_prompt, user_prompt, filename='openai-prompt.yaml'): + data = { + 'model': model, + 'messages': [ + { + 'role': 'system', + 'content': system_prompt + }, + { + 'role': 'user', + 'content': user_prompt + } + ], + 'max_tokens': 200, + 'temperature': 0.7 + } + + with open(filename, 'w', encoding='utf-8') as f: + yaml.dump(data, f, sort_keys=False, allow_unicode=True) + + + + +def preprocess(i): + + env = i['env'] + state = i['state'] + + if 'MLC_OPENAI_CONFIG_PATH' not in env or not os.path.exists(env['MLC_OPENAI_CONFIG_PATH']): + if 'user_prompt' in state: + model = env.get('MLC_OPENAI_MODEL', 'gpt-4o') + user_prompt = state['user_prompt'] + system_prompt = state.get('system_prompt', 'You are an AI agent expected to answer questions correctly') + write_openai_yaml(model, system_prompt, user_prompt, 'tmp-openai-prompt.yaml') + env['MLC_OPENAI_CONFIG_PATH'] = 'tmp-openai-prompt.yaml' + + os_info = i['os_info'] + + env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'], 'openai_call.py')} """ + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + filename='tmp-openai-results.json' + with open(filename, 'r', encoding='utf-8') as f: + data = json.load(f) + + state['MLC_OPENAI_RESPONSE'] = data['content'] + + os_info = i['os_info'] + + return {'return': 0} diff --git a/script/openai-call/meta.yaml b/script/openai-call/meta.yaml new file mode 100644 index 000000000..580162a35 --- /dev/null +++ b/script/openai-call/meta.yaml @@ -0,0 +1,35 @@ +alias: openai-call +automation_alias: script +automation_uid: 5b4e0237da074764 +category: MLC Script Template +deps: + - tags: get,python3 + names: + - python + - python3 +new_env_keys: [] +new_state_keys: +- MLC_OPENAI_RESPONSE +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- query +- openai +- openai-call +- call +input_mapping: + api_key: MLC_OPENAI_API_KEY + config_path: MLC_OPENAI_CONFIG_PATH + system_prompt: MLC_OPENAI_SYSTEM_PROMPT + user_prompt: MLC_OPENAI_USER_PROMPT + model: MLC_OPENAI_MODEL +tests: + run_inputs: [] +uid: 8d341b0a14a64d94 +variations: + openai: + group: api_provider + default: true + env: + MLC_OPENAI_API_URL: 'https://api.openai.com/v1/chat/completions' diff --git a/script/openai-call/openai_call.py b/script/openai-call/openai_call.py new file mode 100644 index 000000000..fa8321ef4 --- /dev/null +++ b/script/openai-call/openai_call.py @@ -0,0 +1,69 @@ +import requests +import yaml +import os +import json + +def openai_call(message=None): + try: + api_key = os.environ['MLC_OPENAI_API_KEY'] + url = os.environ['MLC_OPENAI_API_URL'] + config_path = os.environ.get('MLC_OPENAI_CONFIG_PATH') + + # Load config if it exists + if config_path and os.path.exists(config_path): + try: + with open(config_path, 'r') as file: + data = yaml.safe_load(file) + except Exception as e: + return {"error": f"Error reading config file: {str(e)}"} + + if os.environ.get('MLC_OPENAI_CONFIG_MODIFY', '') == 'yes': + try: + data['messages'][1]['content'] = data['messages'][1]['content'].replace("{{ MESSAGE }}", message or "") + except Exception as e: + return {"error": f"Config format issue: {str(e)}"} + else: + system_prompt = os.environ.get('MLC_OPENAI_SYSTEM_PROMPT', 'You are an AI agent expected to correctly answer the asked question') + user_prompt = message or os.environ.get('MLC_OPENAI_USER_PROMPT', '') + data = { + "model": os.environ.get('MLC_OPENAI_MODEL', 'gpt-4.1'), + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} + ] + } + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}' + } + + + response = requests.post(url, json=data, headers=headers) + response.raise_for_status() + result = response.json() + content = result['choices'][0]['message']['content'] + + with open('tmp-openai-results.json', 'w', encoding='utf-8') as f: + json.dump({'content': content}, f, ensure_ascii=False, indent=2) + + return {"content": content} + + except requests.exceptions.RequestException as e: + return {"error": f"Request error: {str(e)}"} + + except KeyError as e: + return {"error": f"Missing key in response: {str(e)}"} + + except Exception as e: + return {"error": f"Unexpected error: {str(e)}"} + + +def main(): + result = openai_call() + if 'error' in result: + raise Exception(result['error']) + +if __name__ == '__main__': + main() + From a6c5374495602776cf493033556931e3d558b474 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Apr 2025 23:40:04 +0000 Subject: [PATCH 02/55] [Automated Commit] Format Codebase [skip ci] --- script/openai-call/customize.py | 23 +++++++++++++++-------- script/openai-call/openai_call.py | 26 +++++++++++++++----------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/script/openai-call/customize.py b/script/openai-call/customize.py index 9bab431fb..5963e8ebb 100644 --- a/script/openai-call/customize.py +++ b/script/openai-call/customize.py @@ -6,7 +6,9 @@ import yaml -def write_openai_yaml(model, system_prompt, user_prompt, filename='openai-prompt.yaml'): + +def write_openai_yaml(model, system_prompt, user_prompt, + filename='openai-prompt.yaml'): data = { 'model': model, 'messages': [ @@ -27,19 +29,24 @@ def write_openai_yaml(model, system_prompt, user_prompt, filename='openai-prompt yaml.dump(data, f, sort_keys=False, allow_unicode=True) - - def preprocess(i): env = i['env'] state = i['state'] - if 'MLC_OPENAI_CONFIG_PATH' not in env or not os.path.exists(env['MLC_OPENAI_CONFIG_PATH']): + if 'MLC_OPENAI_CONFIG_PATH' not in env or not os.path.exists( + env['MLC_OPENAI_CONFIG_PATH']): if 'user_prompt' in state: model = env.get('MLC_OPENAI_MODEL', 'gpt-4o') user_prompt = state['user_prompt'] - system_prompt = state.get('system_prompt', 'You are an AI agent expected to answer questions correctly') - write_openai_yaml(model, system_prompt, user_prompt, 'tmp-openai-prompt.yaml') + system_prompt = state.get( + 'system_prompt', + 'You are an AI agent expected to answer questions correctly') + write_openai_yaml( + model, + system_prompt, + user_prompt, + 'tmp-openai-prompt.yaml') env['MLC_OPENAI_CONFIG_PATH'] = 'tmp-openai-prompt.yaml' os_info = i['os_info'] @@ -54,10 +61,10 @@ def postprocess(i): env = i['env'] state = i['state'] - filename='tmp-openai-results.json' + filename = 'tmp-openai-results.json' with open(filename, 'r', encoding='utf-8') as f: data = json.load(f) - + state['MLC_OPENAI_RESPONSE'] = data['content'] os_info = i['os_info'] diff --git a/script/openai-call/openai_call.py b/script/openai-call/openai_call.py index fa8321ef4..13a3ce710 100644 --- a/script/openai-call/openai_call.py +++ b/script/openai-call/openai_call.py @@ -3,6 +3,7 @@ import os import json + def openai_call(message=None): try: api_key = os.environ['MLC_OPENAI_API_KEY'] @@ -19,26 +20,29 @@ def openai_call(message=None): if os.environ.get('MLC_OPENAI_CONFIG_MODIFY', '') == 'yes': try: - data['messages'][1]['content'] = data['messages'][1]['content'].replace("{{ MESSAGE }}", message or "") + data['messages'][1]['content'] = data['messages'][1]['content'].replace( + "{{ MESSAGE }}", message or "") except Exception as e: return {"error": f"Config format issue: {str(e)}"} else: - system_prompt = os.environ.get('MLC_OPENAI_SYSTEM_PROMPT', 'You are an AI agent expected to correctly answer the asked question') - user_prompt = message or os.environ.get('MLC_OPENAI_USER_PROMPT', '') + system_prompt = os.environ.get( + 'MLC_OPENAI_SYSTEM_PROMPT', + 'You are an AI agent expected to correctly answer the asked question') + user_prompt = message or os.environ.get( + 'MLC_OPENAI_USER_PROMPT', '') data = { - "model": os.environ.get('MLC_OPENAI_MODEL', 'gpt-4.1'), - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt} - ] - } + "model": os.environ.get('MLC_OPENAI_MODEL', 'gpt-4.1'), + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} + ] + } headers = { 'Content-Type': 'application/json', 'Authorization': f'Bearer {api_key}' } - response = requests.post(url, json=data, headers=headers) response.raise_for_status() result = response.json() @@ -64,6 +68,6 @@ def main(): if 'error' in result: raise Exception(result['error']) + if __name__ == '__main__': main() - From f92df25a2bfed91ffb7d59830e7ad8104d9ece10 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Apr 2025 05:16:59 +0530 Subject: [PATCH 03/55] Added run files for openai call --- script/openai-call/run.bat | 23 +++++++++++++++++++++++ script/openai-call/run.sh | 17 +++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 script/openai-call/run.bat create mode 100644 script/openai-call/run.sh diff --git a/script/openai-call/run.bat b/script/openai-call/run.bat new file mode 100644 index 000000000..12c8a6224 --- /dev/null +++ b/script/openai-call/run.bat @@ -0,0 +1,23 @@ +@echo off +setlocal enabledelayedexpansion + +:: Function to exit if the last command failed +:exit_if_error +if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL% +exit /b 0 + +:: Function to run a command +:run +echo Running: +echo %1 +echo. + +if /I "%MLC_FAKE_RUN%" NEQ "yes" ( + call %1 + call :exit_if_error +) +exit /b 0 + +:: Add your run commands here... +call :run "%MLC_RUN_CMD%" + diff --git a/script/openai-call/run.sh b/script/openai-call/run.sh new file mode 100644 index 000000000..c4542b8c2 --- /dev/null +++ b/script/openai-call/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +function exit_if_error() { + test $? -eq 0 || exit $? +} + +function run() { + echo "Running: " + echo "$1" + echo "" + if [[ ${MLC_FAKE_RUN} != 'yes' ]]; then + eval "$1" + exit_if_error + fi +} + +#Add your run commands here... +run "$MLC_RUN_CMD" From 55173e6851e850fc933ef5feb191ffddeae58c26 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 12:29:45 +0530 Subject: [PATCH 04/55] skip authentication when service account credentials are provided --- script/get-dataset-waymo-calibration/meta.yaml | 1 + script/get-rclone-config/customize.py | 2 +- script/get-rclone-config/meta.yaml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/script/get-dataset-waymo-calibration/meta.yaml b/script/get-dataset-waymo-calibration/meta.yaml index cf9976fa7..c78178cf1 100644 --- a/script/get-dataset-waymo-calibration/meta.yaml +++ b/script/get-dataset-waymo-calibration/meta.yaml @@ -60,6 +60,7 @@ variations: group: run-mode env: MLC_DOWNLOAD_MODE: dry + MLC_BYPASS_RCLONE_AUTH: True dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run \ No newline at end of file diff --git a/script/get-rclone-config/customize.py b/script/get-rclone-config/customize.py index 003d1bf8a..a3df9b1f1 100644 --- a/script/get-rclone-config/customize.py +++ b/script/get-rclone-config/customize.py @@ -19,7 +19,7 @@ def preprocess(i): if env.get('MLC_RCLONE_CONFIG_CMD', '') != '': run_cmds.append(env['MLC_RCLONE_CONFIG_CMD']) - if env.get('MLC_RCLONE_CONNECT_CMD', '') != '': + if env.get('MLC_RCLONE_CONNECT_CMD', '') != '' and not is_true(env.get('MLC_BYPASS_RCLONE_AUTH', '')): run_cmds.append(env['MLC_RCLONE_CONNECT_CMD']) env['MLC_RUN_CMD'] = ' && '.join(run_cmds) diff --git a/script/get-rclone-config/meta.yaml b/script/get-rclone-config/meta.yaml index a7bd8e5b3..8fb2058e2 100644 --- a/script/get-rclone-config/meta.yaml +++ b/script/get-rclone-config/meta.yaml @@ -30,8 +30,8 @@ variations: MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-llama3-1:' waymo: env: - MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-waymo drive config_is_local=false scope=<<>> root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5' - MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-waymo:' + MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc_waymo drive config_is_local=false scope=<<>> root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5' + MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc_waymo:' config-name.#: env: MLC_RCLONE_CONFIG_CMD: 'rclone config create # drive config_is_local=false scope=<<>> root_folder_id=<<>>' From 7ba537be92c90265a2342adc1f133fc90efeb4cf Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 15:33:09 +0530 Subject: [PATCH 05/55] change the remote name --- script/get-dataset-waymo-calibration/meta.yaml | 2 +- script/get-dataset-waymo/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-waymo-calibration/meta.yaml b/script/get-dataset-waymo-calibration/meta.yaml index c78178cf1..96b916764 100644 --- a/script/get-dataset-waymo-calibration/meta.yaml +++ b/script/get-dataset-waymo-calibration/meta.yaml @@ -39,7 +39,7 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_CALIBRATION_PATH - MLC_DOWNLOAD_URL: mlc-waymo:waymo_preprocessed_dataset/kitti_format/testing + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format/testing extra_cache_tags: waymo,dataset force_cache: true names: diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index bfeb56a8c..ef56e4a2e 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -36,7 +36,7 @@ variations: env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH - MLC_DOWNLOAD_URL: mlc-waymo:waymo_preprocessed_dataset/kitti_format + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format extra_cache_tags: waymo,dataset force_cache: true names: From c82bca4145a486e2d4372a97df8ff92fc0046c83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 10:09:32 +0000 Subject: [PATCH 06/55] [Automated Commit] Format Codebase [skip ci] --- script/get-rclone-config/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-rclone-config/customize.py b/script/get-rclone-config/customize.py index a3df9b1f1..92ed8f7d2 100644 --- a/script/get-rclone-config/customize.py +++ b/script/get-rclone-config/customize.py @@ -19,7 +19,8 @@ def preprocess(i): if env.get('MLC_RCLONE_CONFIG_CMD', '') != '': run_cmds.append(env['MLC_RCLONE_CONFIG_CMD']) - if env.get('MLC_RCLONE_CONNECT_CMD', '') != '' and not is_true(env.get('MLC_BYPASS_RCLONE_AUTH', '')): + if env.get('MLC_RCLONE_CONNECT_CMD', '') != '' and not is_true( + env.get('MLC_BYPASS_RCLONE_AUTH', '')): run_cmds.append(env['MLC_RCLONE_CONNECT_CMD']) env['MLC_RUN_CMD'] = ' && '.join(run_cmds) From 01d67d8e7e8998611634ce6e1a75a6207064cae5 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 16:00:28 +0530 Subject: [PATCH 07/55] fix import --- script/get-preprocessed-dataset-openorca/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-preprocessed-dataset-openorca/customize.py b/script/get-preprocessed-dataset-openorca/customize.py index 18b990e09..daeef32f4 100644 --- a/script/get-preprocessed-dataset-openorca/customize.py +++ b/script/get-preprocessed-dataset-openorca/customize.py @@ -1,5 +1,5 @@ from mlc import utils -from mlc.utils import * +from utils import * import os import shutil From 8ea9e55b1605fe662441fb69233f8926ba41dacb Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 16:59:57 +0530 Subject: [PATCH 08/55] path string fix --- script/app-mlperf-inference/customize.py | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 7c74ebd0a..d78499925 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -119,14 +119,14 @@ def postprocess(i): if mode == "accuracy" or mode == "compliance" and env[ 'MLC_MLPERF_LOADGEN_COMPLIANCE_TEST'] == "TEST01": - out_baseline_accuracy_string = f"""> {os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")} """ - out_compliance_accuracy_string = f"""> {os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")} """ + out_baseline_accuracy_string = f"""> '{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}' """ + out_compliance_accuracy_string = f"""> '{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}' """ if model == "resnet50": accuracy_filename = "accuracy-imagenet.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = " --imagenet-val-file " + \ - os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + dataset_args = " --imagenet-val-file '" + \ + os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + "' " accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = " --dtype " + env['MLC_IMAGENET_ACCURACY_DTYPE'] @@ -141,8 +141,8 @@ def postprocess(i): accuracy_filename = "accuracy-openimages.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = " --openimages-dir " + \ - os.getcwd() # just to make the script happy + dataset_args = " --openimages-dir '" + \ + os.getcwd() + "' " # just to make the script happy accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" @@ -165,8 +165,8 @@ def postprocess(i): env['MLC_DATASET_IGBH_SIZE'] + "'" accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" - out_baseline_accuracy_string = f""" --output-file {os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")} """ - out_compliance_accuracy_string = f""" --output-file {os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")} """ + out_baseline_accuracy_string = f""" --output-file '{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}' """ + out_compliance_accuracy_string = f""" --output-file '{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}' """ elif 'stable-diffusion-xl' in model: pass # No compliance check for now @@ -499,9 +499,9 @@ def postprocess(i): test, "run_verification.py") if test == "TEST06": - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {SCRIPT_PATH} -c {COMPLIANCE_DIR} -o {OUTPUT_DIR} --scenario {scenario} --dtype int32" + cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} '{SCRIPT_PATH}' -c '{COMPLIANCE_DIR}' -o '{OUTPUT_DIR}' --scenario {scenario} --dtype int32" else: - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {SCRIPT_PATH} -r {RESULT_DIR} -c {COMPLIANCE_DIR} -o {OUTPUT_DIR}" + cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} '{SCRIPT_PATH}' -r '{RESULT_DIR}' -c '{COMPLIANCE_DIR}' -o '{OUTPUT_DIR}'" print(cmd) os.system(cmd) @@ -524,8 +524,8 @@ def postprocess(i): return { 'return': 1, 'error': 'TEST01 needs accuracy run to be completed first'} - cmd = "cd " + TEST01_DIR + " && bash " + SCRIPT_PATH + " " + os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json") + " " + \ - os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json") + cmd = "cd '" + TEST01_DIR + "' && bash '" + SCRIPT_PATH + "' '" + os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json") + "' '" + \ + os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json") + "' " env['CMD'] = cmd print(cmd) r = automation.run_native_script( @@ -544,8 +544,8 @@ def postprocess(i): baseline_accuracy_file = os.path.join( TEST01_DIR, "mlperf_log_accuracy_baseline.json") - CMD = "cd " + ACCURACY_DIR + " && " + env['MLC_PYTHON_BIN_WITH_PATH'] + ' ' + accuracy_filepath + accuracy_log_file_option_name + \ - baseline_accuracy_file + ' ' + dataset_args + \ + CMD = "cd '" + ACCURACY_DIR + "' && '" + env['MLC_PYTHON_BIN_WITH_PATH'] + "' '" + accuracy_filepath + "' " + accuracy_log_file_option_name + " '" + \ + baseline_accuracy_file + "' " + dataset_args + \ datatype_option + out_baseline_accuracy_string env['CMD'] = CMD @@ -558,8 +558,8 @@ def postprocess(i): return {'return': 1, 'error': f"{baseline_accuracy_file} is empty"} - CMD = "cd " + ACCURACY_DIR + " && " + env['MLC_PYTHON_BIN_WITH_PATH'] + ' ' + accuracy_filepath + accuracy_log_file_option_name + \ - os.path.join(TEST01_DIR, "mlperf_log_accuracy.json") + \ + CMD = "cd '" + ACCURACY_DIR + "' && '" + env['MLC_PYTHON_BIN_WITH_PATH'] + "' '" + accuracy_filepath + "' " + accuracy_log_file_option_name + " '" +\ + os.path.join(TEST01_DIR, "mlperf_log_accuracy.json") + "' " + \ dataset_args + datatype_option + out_compliance_accuracy_string env['CMD'] = CMD From c2cdade828dd175685145b2c5c1843bd1253a091 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Apr 2025 17:15:10 +0530 Subject: [PATCH 09/55] Fix dgl version for mlperf inference rgat --- script/app-mlperf-inference-mlcommons-python/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index fe7793e2d..a0380fec2 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1326,6 +1326,8 @@ variations: _find_links_url.: - MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL - tags: get,generic-python-lib,_package.dgl + version_max: '2.4.0' + version_max_usable: '2.4.0' update_tags_from_env_with_prefix: _find_links_url.: - MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL From 5e53ee4734e1b7bbc03cdcfa4f682bf334cec965 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 17:57:32 +0530 Subject: [PATCH 10/55] fix quotes --- script/app-mlperf-inference/customize.py | 33 +++++++++++------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index d78499925..1bb88ad96 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -55,6 +55,7 @@ def postprocess(i): os_info = i['os_info'] xsep = '^' if os_info['platform'] == 'windows' else '\\' + q = '"' if os_info['platform'] == 'windows' else "'" env = i['env'] inp = i['input'] @@ -119,14 +120,13 @@ def postprocess(i): if mode == "accuracy" or mode == "compliance" and env[ 'MLC_MLPERF_LOADGEN_COMPLIANCE_TEST'] == "TEST01": - out_baseline_accuracy_string = f"""> '{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}' """ - out_compliance_accuracy_string = f"""> '{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}' """ + out_baseline_accuracy_string = f"""> {q}{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}{q} """ + out_compliance_accuracy_string = f"""> {q}{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}{q} """ if model == "resnet50": accuracy_filename = "accuracy-imagenet.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = " --imagenet-val-file '" + \ - os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + "' " + dataset_args = f" --imagenet-val-file {q}{os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt")}{q} " accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = " --dtype " + env['MLC_IMAGENET_ACCURACY_DTYPE'] @@ -141,8 +141,7 @@ def postprocess(i): accuracy_filename = "accuracy-openimages.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = " --openimages-dir '" + \ - os.getcwd() + "' " # just to make the script happy + dataset_args = f" --openimages-dir {q}{os.getcwd()}{q} " # just to make the script happy accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" @@ -165,8 +164,8 @@ def postprocess(i): env['MLC_DATASET_IGBH_SIZE'] + "'" accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" - out_baseline_accuracy_string = f""" --output-file '{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}' """ - out_compliance_accuracy_string = f""" --output-file '{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}' """ + out_baseline_accuracy_string = f""" --output-file {q}{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}{q} """ + out_compliance_accuracy_string = f""" --output-file {q}{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}{q} """ elif 'stable-diffusion-xl' in model: pass # No compliance check for now @@ -499,9 +498,9 @@ def postprocess(i): test, "run_verification.py") if test == "TEST06": - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} '{SCRIPT_PATH}' -c '{COMPLIANCE_DIR}' -o '{OUTPUT_DIR}' --scenario {scenario} --dtype int32" + cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q} --scenario {scenario} --dtype int32" else: - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} '{SCRIPT_PATH}' -r '{RESULT_DIR}' -c '{COMPLIANCE_DIR}' -o '{OUTPUT_DIR}'" + cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -r {q}{RESULT_DIR}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q}" print(cmd) os.system(cmd) @@ -524,8 +523,7 @@ def postprocess(i): return { 'return': 1, 'error': 'TEST01 needs accuracy run to be completed first'} - cmd = "cd '" + TEST01_DIR + "' && bash '" + SCRIPT_PATH + "' '" + os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json") + "' '" + \ - os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json") + "' " + cmd = f"cd {q}{TEST01_DIR}{q} && bash {q}{SCRIPT_PATH}{q} {q}{os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json")}{q} {q}{os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json")}{q} " env['CMD'] = cmd print(cmd) r = automation.run_native_script( @@ -544,9 +542,8 @@ def postprocess(i): baseline_accuracy_file = os.path.join( TEST01_DIR, "mlperf_log_accuracy_baseline.json") - CMD = "cd '" + ACCURACY_DIR + "' && '" + env['MLC_PYTHON_BIN_WITH_PATH'] + "' '" + accuracy_filepath + "' " + accuracy_log_file_option_name + " '" + \ - baseline_accuracy_file + "' " + dataset_args + \ - datatype_option + out_baseline_accuracy_string + CMD = f"""cd {q}{ACCURACY_DIR}{q} && {q}{env['MLC_PYTHON_BIN_WITH_PATH']}{q} {q}{accuracy_filepath}{q} \ +{accuracy_log_file_option_name} {q}{baseline_accuracy_file}{q} {dataset_args} {datatype_option} {out_baseline_accuracy_string} """ env['CMD'] = CMD r = automation.run_native_script( @@ -558,9 +555,9 @@ def postprocess(i): return {'return': 1, 'error': f"{baseline_accuracy_file} is empty"} - CMD = "cd '" + ACCURACY_DIR + "' && '" + env['MLC_PYTHON_BIN_WITH_PATH'] + "' '" + accuracy_filepath + "' " + accuracy_log_file_option_name + " '" +\ - os.path.join(TEST01_DIR, "mlperf_log_accuracy.json") + "' " + \ - dataset_args + datatype_option + out_compliance_accuracy_string + CMD = f"""cd {q}{ACCURACY_DIR}{q} && {q}{env['MLC_PYTHON_BIN_WITH_PATH']}{q} {q}{accuracy_filepath}{q} \ +{accuracy_log_file_option_name} {q}{os.path.join(TEST01_DIR, "mlperf_log_accuracy.json")}{q} {dataset_args} {datatype_option} \ +{out_compliance_accuracy_string} """ env['CMD'] = CMD r = automation.run_native_script( From c8c969bd152fff73ffb759cf4dff6e329bb870c3 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 18:13:38 +0530 Subject: [PATCH 11/55] add space in MLC repo folder --- .../test-mlperf-inference-resnet50-closed-division.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index f7afd28c0..12d921069 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -64,7 +64,15 @@ jobs: if: matrix.os == 'windows-latest' run: | git config --system core.longpaths true - + - name: Export MLC_REPOS in Linux or Mac + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + run: | + echo "MLC_REPOS=$HOME/gh action/mlc" >> $GITHUB_ENV + - name: Export MLC_REPOS in Windows + if: matrix.os == 'windows-latest' + run: | + $mlcrepos = "${env:USERPROFILE}\gh action\mlc" + "MLC_REPOS=$mlcRepos" | Out-File -FilePath $env:GITHUB_ENV -Append - name: Install mlcflow run: | pip install mlcflow From 432a9cda53689f26f54a70d5b7c0cb2adf39d1d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Apr 2025 12:43:56 +0000 Subject: [PATCH 12/55] [Automated Commit] Format Codebase [skip ci] --- script/app-mlperf-inference/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 1bb88ad96..3f6b453f5 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -141,7 +141,8 @@ def postprocess(i): accuracy_filename = "accuracy-openimages.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = f" --openimages-dir {q}{os.getcwd()}{q} " # just to make the script happy + # just to make the script happy + dataset_args = f" --openimages-dir {q}{os.getcwd()}{q} " accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" From 3359e6ab15147abd81372a5b70d0251452d4e73f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 18:14:27 +0530 Subject: [PATCH 13/55] test commit --- .../workflows/test-mlperf-inference-resnet50-closed-division.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 12d921069..abc923e8e 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -55,7 +55,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: From 713ce3302bb3dce50112ec6f7b7ce2cc34371352 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 18:37:23 +0530 Subject: [PATCH 14/55] better handling of fstring --- script/app-mlperf-inference/customize.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 3f6b453f5..e67e35cf1 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -126,7 +126,7 @@ def postprocess(i): accuracy_filename = "accuracy-imagenet.py" accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) - dataset_args = f" --imagenet-val-file {q}{os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt")}{q} " + dataset_args = f""" --imagenet-val-file {q}{os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt")}{q} """ accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = " --dtype " + env['MLC_IMAGENET_ACCURACY_DTYPE'] @@ -142,7 +142,7 @@ def postprocess(i): accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename) # just to make the script happy - dataset_args = f" --openimages-dir {q}{os.getcwd()}{q} " + dataset_args = f""" --openimages-dir {q}{os.getcwd()}{q} """ accuracy_log_file_option_name = " --mlperf-accuracy-file " datatype_option = "" @@ -499,9 +499,9 @@ def postprocess(i): test, "run_verification.py") if test == "TEST06": - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q} --scenario {scenario} --dtype int32" + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q} --scenario {scenario} --dtype int32""" else: - cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -r {q}{RESULT_DIR}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q}" + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -r {q}{RESULT_DIR}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q}""" print(cmd) os.system(cmd) @@ -524,7 +524,7 @@ def postprocess(i): return { 'return': 1, 'error': 'TEST01 needs accuracy run to be completed first'} - cmd = f"cd {q}{TEST01_DIR}{q} && bash {q}{SCRIPT_PATH}{q} {q}{os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json")}{q} {q}{os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json")}{q} " + cmd = f"""cd {q}{TEST01_DIR}{q} && bash {q}{SCRIPT_PATH}{q} {q}{os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json")}{q} {q}{os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json")}{q} """ env['CMD'] = cmd print(cmd) r = automation.run_native_script( From 1595a019c8c2af36d8457dd8cf5ad20ec0f59a2d Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 20:04:35 +0530 Subject: [PATCH 15/55] test commit - closed division run on pull request target --- .../test-mlperf-inference-resnet50-closed-division.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index abc923e8e..a680f3ed9 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -4,6 +4,12 @@ on: schedule: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow + pull_request_target: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml' + - '**' + - '!**.md' jobs: fetch-secret: runs-on: ubuntu-latest From 128d7980ac784096e3220e7f692b5e2f4a8190f4 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 22 Apr 2025 20:06:58 +0530 Subject: [PATCH 16/55] revert test commit --- .../test-mlperf-inference-resnet50-closed-division.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index a680f3ed9..abc923e8e 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -4,12 +4,6 @@ on: schedule: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow - pull_request_target: - branches: [ "main", "dev" ] - paths: - - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml' - - '**' - - '!**.md' jobs: fetch-secret: runs-on: ubuntu-latest From bdb6cff684a6b451f73216acd43cc4ee6b387724 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 10:25:53 +0530 Subject: [PATCH 17/55] fix command generation --- script/pull-git-repo/run.bat | 2 +- script/pull-git-repo/run.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/script/pull-git-repo/run.bat b/script/pull-git-repo/run.bat index 830588501..30de3de31 100644 --- a/script/pull-git-repo/run.bat +++ b/script/pull-git-repo/run.bat @@ -7,7 +7,7 @@ set "SCRIPT_DIR=%MLC_TMP_CURRENT_SCRIPT_PATH%" REM Change to the specified path set "path=%MLC_GIT_CHECKOUT_PATH%" -echo cd %path% +echo cd "%path%" cd /d "%path%" if errorlevel 1 ( diff --git a/script/pull-git-repo/run.sh b/script/pull-git-repo/run.sh index c75f5b45d..1fd5f82d8 100644 --- a/script/pull-git-repo/run.sh +++ b/script/pull-git-repo/run.sh @@ -3,10 +3,10 @@ CUR_DIR=$PWD SCRIPT_DIR=${MLC_TMP_CURRENT_SCRIPT_PATH} -path=${MLC_GIT_CHECKOUT_PATH} -echo "cd $path" +path="${MLC_GIT_CHECKOUT_PATH}" +echo "cd \"$path\"" -cd $path +cd "$path" test $? -eq 0 || exit $? echo ${MLC_GIT_PULL_CMD} From da86f8db5f220fd1bf9e168dde4f6a32f20969af Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 10:45:21 +0530 Subject: [PATCH 18/55] test commit - fix command generation --- .../test-mlperf-inference-resnet50-closed-division.yml | 6 ++++++ script/pull-git-repo/run.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index abc923e8e..adba9a256 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -4,6 +4,12 @@ on: schedule: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow + pull_request_target: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' + - '**' + - '!**.md' jobs: fetch-secret: runs-on: ubuntu-latest diff --git a/script/pull-git-repo/run.sh b/script/pull-git-repo/run.sh index 1fd5f82d8..0179142e4 100644 --- a/script/pull-git-repo/run.sh +++ b/script/pull-git-repo/run.sh @@ -14,4 +14,4 @@ eval ${MLC_GIT_PULL_CMD} #don't fail if there are local changes #test $? -eq 0 || exit $? -cd $CUR_DIR +cd "$CUR_DIR" From e0eab4131e4ced7a30efc4aee2a47d3ff6b87458 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 10:51:51 +0530 Subject: [PATCH 19/55] revert the workflow change --- .../test-mlperf-inference-resnet50-closed-division.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index adba9a256..abc923e8e 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -4,12 +4,6 @@ on: schedule: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow - pull_request_target: - branches: [ "main", "dev" ] - paths: - - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - - '**' - - '!**.md' jobs: fetch-secret: runs-on: ubuntu-latest From e67da9f53a53e1caa8b661becd9159a878592ded Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 11:18:47 +0530 Subject: [PATCH 20/55] fix command generation - paths with space --- script/get-generic-python-lib/install.bat | 4 ++-- script/get-generic-python-lib/run.bat | 4 ++-- script/get-generic-python-lib/run.sh | 4 ++-- script/get-generic-python-lib/validate_cache.bat | 4 ++-- script/get-generic-python-lib/validate_cache.sh | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/script/get-generic-python-lib/install.bat b/script/get-generic-python-lib/install.bat index e74450c72..7feef97b7 100644 --- a/script/get-generic-python-lib/install.bat +++ b/script/get-generic-python-lib/install.bat @@ -2,12 +2,12 @@ echo. if NOT "%MLC_GENERIC_PYTHON_PIP_URL%" == "" ( - %MLC_PYTHON_BIN_WITH_PATH% -m pip install %MLC_GENERIC_PYTHON_PIP_URL% %MLC_GENERIC_PYTHON_PIP_EXTRA% + "%MLC_PYTHON_BIN_WITH_PATH%" -m pip install %MLC_GENERIC_PYTHON_PIP_URL% %MLC_GENERIC_PYTHON_PIP_EXTRA% IF %ERRORLEVEL% NEQ 0 EXIT 1 ) else ( - %MLC_PYTHON_BIN_WITH_PATH% -m pip install %MLC_GENERIC_PYTHON_PACKAGE_NAME%%MLC_TMP_PIP_VERSION_STRING% %MLC_GENERIC_PYTHON_PIP_EXTRA% + "%MLC_PYTHON_BIN_WITH_PATH%" -m pip install %MLC_GENERIC_PYTHON_PACKAGE_NAME%%MLC_TMP_PIP_VERSION_STRING% %MLC_GENERIC_PYTHON_PIP_EXTRA% IF %ERRORLEVEL% NEQ 0 EXIT 1 ) diff --git a/script/get-generic-python-lib/run.bat b/script/get-generic-python-lib/run.bat index 17e27e030..af803d720 100644 --- a/script/get-generic-python-lib/run.bat +++ b/script/get-generic-python-lib/run.bat @@ -1,4 +1,4 @@ -IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET MLC_TMP_CURRENT_SCRIPT_PATH=%CD% +IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET "MLC_TMP_CURRENT_SCRIPT_PATH=%CD%" -%MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py +%MLC_PYTHON_BIN_WITH_PATH% "%MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py" IF %ERRORLEVEL% NEQ 0 EXIT 1 diff --git a/script/get-generic-python-lib/run.sh b/script/get-generic-python-lib/run.sh index 2df36823d..9a98e610e 100644 --- a/script/get-generic-python-lib/run.sh +++ b/script/get-generic-python-lib/run.sh @@ -1,7 +1,7 @@ #!/bin/bash -MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} +MLC_TMP_CURRENT_SCRIPT_PATH="${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD}" -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/detect-version.py +"${MLC_PYTHON_BIN_WITH_PATH}" "${MLC_TMP_CURRENT_SCRIPT_PATH}/detect-version.py" test $? -eq 0 || exit $? exit 0 diff --git a/script/get-generic-python-lib/validate_cache.bat b/script/get-generic-python-lib/validate_cache.bat index 17e27e030..c3bac688b 100644 --- a/script/get-generic-python-lib/validate_cache.bat +++ b/script/get-generic-python-lib/validate_cache.bat @@ -1,4 +1,4 @@ -IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET MLC_TMP_CURRENT_SCRIPT_PATH=%CD% +IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET "MLC_TMP_CURRENT_SCRIPT_PATH=%CD%" -%MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py +"%MLC_PYTHON_BIN_WITH_PATH%" "%MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py" IF %ERRORLEVEL% NEQ 0 EXIT 1 diff --git a/script/get-generic-python-lib/validate_cache.sh b/script/get-generic-python-lib/validate_cache.sh index 2df36823d..9a98e610e 100644 --- a/script/get-generic-python-lib/validate_cache.sh +++ b/script/get-generic-python-lib/validate_cache.sh @@ -1,7 +1,7 @@ #!/bin/bash -MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} +MLC_TMP_CURRENT_SCRIPT_PATH="${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD}" -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/detect-version.py +"${MLC_PYTHON_BIN_WITH_PATH}" "${MLC_TMP_CURRENT_SCRIPT_PATH}/detect-version.py" test $? -eq 0 || exit $? exit 0 From c881e13d1b1021f0cc983f0c282616136588092f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 11:39:11 +0530 Subject: [PATCH 21/55] fix for handling space --- script/get-preprocessed-dataset-imagenet/run.bat | 2 +- script/get-preprocessed-dataset-imagenet/run.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-preprocessed-dataset-imagenet/run.bat b/script/get-preprocessed-dataset-imagenet/run.bat index bdeca68fd..3ee17691b 100644 --- a/script/get-preprocessed-dataset-imagenet/run.bat +++ b/script/get-preprocessed-dataset-imagenet/run.bat @@ -1,4 +1,4 @@ @echo off -%MLC_PYTHON_BIN% %MLC_TMP_CURRENT_SCRIPT_PATH%\preprocess.py +"%MLC_PYTHON_BIN%" "%MLC_TMP_CURRENT_SCRIPT_PATH%\preprocess.py" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/script/get-preprocessed-dataset-imagenet/run.sh b/script/get-preprocessed-dataset-imagenet/run.sh index 04b4b3cff..436b11582 100644 --- a/script/get-preprocessed-dataset-imagenet/run.sh +++ b/script/get-preprocessed-dataset-imagenet/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ ! -z ${MLC_IMAGENET_PREPROCESSED_PATH+x} ]; then +if [ ! -z "${MLC_IMAGENET_PREPROCESSED_PATH+x}" ]; then exit 0 fi -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/preprocess.py +"${MLC_PYTHON_BIN_WITH_PATH}" "${MLC_TMP_CURRENT_SCRIPT_PATH}/preprocess.py" test $? -eq 0 || exit 1 From 9ceafe43865e05169d8f717d35c4ee25f4dec887 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 12:47:34 +0530 Subject: [PATCH 22/55] fixes for path issues --- script/benchmark-program/customize.py | 3 +-- script/compile-program/run.bat | 8 ++++---- script/compile-program/run.sh | 10 +++++----- script/extract-file/customize.py | 4 ++-- script/extract-file/run.bat | 10 +++++----- script/get-preprocessed-dataset-criteo/customize.py | 2 +- .../preprocess_multihot.sh | 10 +++++----- 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index a355e8248..de0fac475 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -43,8 +43,7 @@ def preprocess(i): if env.get('MLC_RUN_DIR', '') == '': env['MLC_RUN_DIR'] = os.getcwd() - env['MLC_RUN_CMD'] = MLC_RUN_PREFIX + ' ' + os.path.join( - env['MLC_RUN_DIR'], env['MLC_BIN_NAME']) + ' ' + env['MLC_RUN_SUFFIX'] + env['MLC_RUN_CMD'] = f"""{MLC_RUN_PREFIX} {q}{os.path.join(env['MLC_RUN_DIR'], env['MLC_BIN_NAME'])}{q} {env['MLC_RUN_SUFFIX']}""" x = env.get('MLC_RUN_PREFIX0', '') if x != '': diff --git a/script/compile-program/run.bat b/script/compile-program/run.bat index 8a9e5436d..52bbaba75 100644 --- a/script/compile-program/run.bat +++ b/script/compile-program/run.bat @@ -16,20 +16,20 @@ echo. echo Compiling source files ... echo. -if not exist %RUN_DIR% mkdir %RUN_DIR% +if not exist "%RUN_DIR%" mkdir "%RUN_DIR%" -cd %MLC_SOURCE_FOLDER_PATH% +cd "%MLC_SOURCE_FOLDER_PATH%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% if not "%MLC_C_SOURCE_FILES%" == "" ( echo %MLC_C_COMPILER_WITH_PATH% %MLC_C_COMPILER_FLAGS% %MLC_C_INCLUDE_PATH% %MLC_C_SOURCE_FILES% %MLC_LD_LIBRARY_PATH% %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" - "%MLC_C_COMPILER_WITH_PATH%" %MLC_C_COMPILER_FLAGS% %MLC_C_INCLUDE_PATH% %MLC_C_SOURCE_FILES% %MLC_LD_LIBRARY_PATH% %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" + "%MLC_C_COMPILER_WITH_PATH%" %MLC_C_COMPILER_FLAGS% "%MLC_C_INCLUDE_PATH%" %MLC_C_SOURCE_FILES% "%MLC_LD_LIBRARY_PATH%" %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% ) if not "%MLC_CXX_SOURCE_FILES%" == "" ( echo %MLC_CXX_COMPILER_WITH_PATH% %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% %MLC_CPLUS_INCLUDE_PATH% %MLC_LD_LIBRARY_PATH% %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" - "%MLC_CXX_COMPILER_WITH_PATH%" %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% %MLC_CPLUS_INCLUDE_PATH% %MLC_LD_LIBRARY_PATH% %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" + "%MLC_CXX_COMPILER_WITH_PATH%" %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% "%MLC_CPLUS_INCLUDE_PATH%" "%MLC_LD_LIBRARY_PATH%" %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% ) diff --git a/script/compile-program/run.sh b/script/compile-program/run.sh index 68045945d..171b99070 100644 --- a/script/compile-program/run.sh +++ b/script/compile-program/run.sh @@ -3,16 +3,16 @@ # Compile BIN_NAME=${MLC_BIN_NAME:-run.out} -RUN_DIR=${MLC_RUN_DIR:-.} +RUN_DIR="${MLC_RUN_DIR:-.}" echo "RUN_DIR=$RUN_DIR" if [[ ${MLC_SKIP_RECOMPILE} == "yes" ]]; then - if [ -f ${RUN_DIR}/${BIN_NAME} ]; then + if [ -f "${RUN_DIR}/${BIN_NAME}" ]; then exit 0 fi fi -rm -f ${RUN_DIR}/${BIN_NAME} +rm -f "${RUN_DIR}/${BIN_NAME}" if [ -z "${MLC_SOURCE_FOLDER_PATH}" ]; then echo "No source directory (MLC_SOURCE_FOLDER_PATH} specified" @@ -28,13 +28,13 @@ echo "" echo "Checking compiler version ..." echo "" -${MLC_C_COMPILER_WITH_PATH} ${MLC_C_COMPILER_FLAG_VERSION} +"${MLC_C_COMPILER_WITH_PATH}" ${MLC_C_COMPILER_FLAG_VERSION} echo "" echo "Compiling source files ..." echo "" -cd ${MLC_SOURCE_FOLDER_PATH} +cd "${MLC_SOURCE_FOLDER_PATH}" test $? -eq 0 || exit 1 IFS=';' read -ra FILES <<< "${MLC_C_SOURCE_FILES}" diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index f6f041dbc..89b57c927 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -117,10 +117,10 @@ def preprocess(i): y = '"' if ' ' in extract_to_folder else '' # env['MLC_EXTRACT_TOOL_OPTIONS'] = ' --one-top-level='+ env['MLC_EXTRACT_TO_FOLDER'] + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') - env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + extract_to_folder + \ + env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + q + extract_to_folder + q + \ y + ' ' + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') env['MLC_EXTRACT_PRE_CMD'] = 'mkdir ' + x + ' ' + \ - y + extract_to_folder + y + ' ' + xsep + ' ' + y + q + extract_to_folder + q + y + ' ' + xsep + ' ' env['MLC_EXTRACT_EXTRACTED_FILENAME'] = extract_to_folder elif 'unzip' in env['MLC_EXTRACT_TOOL']: diff --git a/script/extract-file/run.bat b/script/extract-file/run.bat index 2a2727965..bfc066533 100644 --- a/script/extract-file/run.bat +++ b/script/extract-file/run.bat @@ -12,10 +12,10 @@ if exist "%MLC_EXTRACT_EXTRACTED_FILENAME%" ( echo. echo %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% - cmd /c %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% + cmd /c "%MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD%" IF !ERRORLEVEL! NEQ 0 ( set require_extract=1 - del /Q %MLC_EXTRACT_EXTRACTED_FILENAME% + del /Q "%MLC_EXTRACT_EXTRACTED_FILENAME%" ) ) @@ -23,17 +23,17 @@ if "!require_extract!" == "1" ( if not "%MLC_EXTRACT_CMD0%" == "" ( echo. echo %MLC_EXTRACT_CMD0% - cmd /c %MLC_EXTRACT_CMD0% + cmd /c "%MLC_EXTRACT_CMD0%" IF !ERRORLEVEL! NEQ 0 EXIT 1 ) echo. echo %MLC_EXTRACT_CMD% - cmd /c %MLC_EXTRACT_CMD% + cmd /c "%MLC_EXTRACT_CMD%" IF !ERRORLEVEL! NEQ 0 EXIT 1 echo. echo %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% - cmd /c %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% + cmd /c "%MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD%" IF !ERRORLEVEL! NEQ 0 EXIT 1 ) diff --git a/script/get-preprocessed-dataset-criteo/customize.py b/script/get-preprocessed-dataset-criteo/customize.py index 723e209bf..668152284 100644 --- a/script/get-preprocessed-dataset-criteo/customize.py +++ b/script/get-preprocessed-dataset-criteo/customize.py @@ -32,7 +32,7 @@ def preprocess(i): "recommendation_v2", "torchrec_dlrm", "scripts") - env['MLC_RUN_CMD'] = f'cd {run_dir} && bash ./process_Criteo_1TB_Click_Logs_dataset.sh {dataset_path} {tmp_dir} {output_dir} ' + env['MLC_RUN_CMD'] = f"""cd '{run_dir}' && bash ./process_Criteo_1TB_Click_Logs_dataset.sh '{dataset_path}' '{tmp_dir}' '{output_dir}' """ print("Using MLCommons Training source from '" + env['MLC_MLPERF_TRAINING_SOURCE'] + "'") diff --git a/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh b/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh index dadf7566c..e38ecd87c 100644 --- a/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh +++ b/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh @@ -1,9 +1,9 @@ #!/bin/bash -cd ${MLC_MLPERF_TRAINING_SOURCE}/recommendation_v2_torchrec_dlrm/ -${MLC_PYTHON_BIN_WITH_PATH} materialize_synthetic_multihot_dataset.py \ - --in_memory_binary_criteo_path $PREPROCESSED_CRITEO_1TB_CLICK_LOGS_DATASET_PATH \ - --output_path $MATERIALIZED_DATASET_PATH \ +cd "${MLC_MLPERF_TRAINING_SOURCE}/recommendation_v2_torchrec_dlrm/" +"${MLC_PYTHON_BIN_WITH_PATH}" "materialize_synthetic_multihot_dataset.py \ + --in_memory_binary_criteo_path ${PREPROCESSED_CRITEO_1TB_CLICK_LOGS_DATASET_PATH} \ + --output_path ${MATERIALIZED_DATASET_PATH} \ --num_embeddings_per_feature 40000000,39060,17295,7424,20265,3,7122,1543,63,40000000,3067956,405282,10,2209,11938,155,4,976,14,40000000,40000000,40000000,590152,12973,108,36 \ --multi_hot_sizes 3,2,1,2,6,1,1,1,1,7,3,8,1,6,9,5,1,1,1,12,100,27,10,3,1,1 \ - --multi_hot_distribution_type uniform + --multi_hot_distribution_type uniform" test $? -eq 0 || exit $? From a6316bcbde6b65bae33cf5b3b45e3147592440bf Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 12:55:37 +0530 Subject: [PATCH 23/55] test commit --- script/extract-file/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index 89b57c927..f6f041dbc 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -117,10 +117,10 @@ def preprocess(i): y = '"' if ' ' in extract_to_folder else '' # env['MLC_EXTRACT_TOOL_OPTIONS'] = ' --one-top-level='+ env['MLC_EXTRACT_TO_FOLDER'] + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') - env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + q + extract_to_folder + q + \ + env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + extract_to_folder + \ y + ' ' + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') env['MLC_EXTRACT_PRE_CMD'] = 'mkdir ' + x + ' ' + \ - y + q + extract_to_folder + q + y + ' ' + xsep + ' ' + y + extract_to_folder + y + ' ' + xsep + ' ' env['MLC_EXTRACT_EXTRACTED_FILENAME'] = extract_to_folder elif 'unzip' in env['MLC_EXTRACT_TOOL']: From dcd43fb21d3f9c9372fa3d31c10caad9d0684ddb Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:11:56 +0530 Subject: [PATCH 24/55] commit for command formation --- .../customize.py | 217 ++++++++---------- script/benchmark-program/run.bat | 2 +- script/compile-program/customize.py | 21 +- 3 files changed, 114 insertions(+), 126 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index bbdff3f7c..2cedc7d9c 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -82,7 +82,7 @@ def preprocess(i): env['MLC_MLPERF_CONF'] = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], "mlperf.conf") - x = "" if os_info['platform'] == 'windows' else "'" + x = '"' if os_info['platform'] == 'windows' else "'" inference_src_version = env.get('MLC_MLPERF_INFERENCE_SOURCE_VERSION', '') version_tuple = None @@ -129,11 +129,10 @@ def preprocess(i): ml_model_name = env['MLC_MODEL'] if 'MLC_MLPERF_USER_CONF' in env: user_conf_path = env['MLC_MLPERF_USER_CONF'] - x = "" if os_info['platform'] == 'windows' else "'" if 'llama2-70b' in env['MLC_MODEL'] or "mixtral-8x7b" in env["MLC_MODEL"] or "llama3" in env["MLC_MODEL"]: - scenario_extra_options += " --user-conf " + x + user_conf_path + x + scenario_extra_options += f""" --user-conf {x}{user_conf_path}{x} """ else: - scenario_extra_options += " --user_conf " + x + user_conf_path + x + scenario_extra_options += f""" --user_conf {x}{user_conf_path}{x} """ mode = env['MLC_MLPERF_LOADGEN_MODE'] mode_extra_options = "" @@ -142,16 +141,13 @@ def preprocess(i): 'resnet50', 'retinanet']: # dataset_options = " --use_preprocessed_dataset --preprocessed_dir "+env['MLC_DATASET_PREPROCESSED_PATH'] if env.get('MLC_MLPERF_LAST_RELEASE') not in ["v2.0", "v2.1"]: - dataset_options = " --use_preprocessed_dataset --cache_dir " + \ - env['MLC_DATASET_PREPROCESSED_PATH'] + dataset_options = f""" --use_preprocessed_dataset --cache_dir {x}{env['MLC_DATASET_PREPROCESSED_PATH']}{x}""" else: dataset_options = "" if env['MLC_MODEL'] == "retinanet": - dataset_options += " --dataset-list " + \ - env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] + dataset_options += f""" --dataset-list {x}{env['MLC_DATASET_ANNOTATIONS_FILE_PATH']}{x}""" elif env['MLC_MODEL'] == "resnet50": - dataset_options += " --dataset-list " + \ - os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + dataset_options += f""" --dataset-list {x}{os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt")}{x}""" env['DATA_DIR'] = env.get('MLC_DATASET_PREPROCESSED_PATH') else: if 'MLC_DATASET_PREPROCESSED_PATH' in env: @@ -176,7 +172,7 @@ def preprocess(i): elif mode == "compliance": audit_full_path = env['MLC_MLPERF_INFERENCE_AUDIT_PATH'] - mode_extra_options = " --audit '" + audit_full_path + "'" + mode_extra_options = f""" --audit {x}{audit_full_path}{x} """ if env.get('MLC_MLPERF_OUTPUT_DIR', '') == '': env['MLC_MLPERF_OUTPUT_DIR'] = os.getcwd() @@ -222,20 +218,16 @@ def get_run_cmd_reference( device = env['MLC_MLPERF_DEVICE'] if env['MLC_MLPERF_DEVICE'] not in [ "gpu", "rocm"] else "cuda" + x = '"' if os_info['platform'] == 'windows' else "'" + if env['MLC_MODEL'] in ["gptj-99", "gptj-99.9"]: env['RUN_DIR'] = os.path.join( env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "gpt-j") if env.get('MLC_NETWORK_LOADGEN', '') != "lon": - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + \ - " main.py --model-path=" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + ' --dataset-path=' + env['MLC_DATASET_EVAL_PATH'] + " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'] + \ - scenario_extra_options + mode_extra_options + dataset_options + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py --model-path {x}{env['MLC_ML_MODEL_FILE_WITH_PATH']}{x} --dataset-path {x}{env['MLC_DATASET_EVAL_PATH']}{x} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} --dtype {env['MLC_MLPERF_MODEL_PRECISION']} {scenario_extra_options} {mode_extra_options} {dataset_options} """ else: - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + \ - " main.py" + ' --dataset-path=' + env['MLC_DATASET_EVAL_PATH'] + " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'] + \ - scenario_extra_options + mode_extra_options + dataset_options + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py --dataset-path {x}{env['MLC_DATASET_EVAL_PATH']}{x} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} --dtype {env['MLC_MLPERF_MODEL_PRECISION']} {scenario_extra_options} {mode_extra_options} {dataset_options} """ cmd = cmd.replace("--count", "--max_examples") if env['MLC_MLPERF_DEVICE'] == "gpu": gpu_options = " --gpu" @@ -282,20 +274,13 @@ def get_run_cmd_reference( env['LOG_PATH'] = env['MLC_MLPERF_OUTPUT_DIR'] - extra_options = " --output " + env['MLC_MLPERF_OUTPUT_DIR'] + " --model-name resnet50 --dataset " + env['MLC_MLPERF_VISION_DATASET_OPTION'] + f""" --max-batchsize {env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '1')}""" + \ - " --dataset-path " + env['MLC_DATASET_PREPROCESSED_PATH'] + " --model " + env['MODEL_FILE'] + \ - " --preprocessed_dir " + env['MLC_DATASET_PREPROCESSED_PATH'] + extra_options = f""" --output {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} --model-name resnet50 --dataset {env['MLC_MLPERF_VISION_DATASET_OPTION']} --max-batchsize {env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '1')} --dataset-path {x}{env['MLC_DATASET_PREPROCESSED_PATH']}{x} --model {x}{env['MODEL_FILE']}{x} --preprocessed_dir {x}{env['MLC_DATASET_PREPROCESSED_PATH']}{x}""" if env.get('MLC_MLPERF_DEVICE') == "tpu": - cmd = "cd '" + os.path.join(env['RUN_DIR'], "python") + "' && " + env.get('MLC_SUDO', "") + " " + env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " +\ - "--backend " + env['MLC_MLPERF_BACKEND'] + " --scenario=" + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " --device tpu " + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + \ - mode_extra_options + dataset_options + extra_options + cmd = f"""cd {x}{os.path.join(env['RUN_DIR'], 'python')}{x} && {env.get('MLC_SUDO', '')} {x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py --backend {env['MLC_MLPERF_BACKEND']} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']} --device tpu {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']}{scenario_extra_options}{mode_extra_options}{dataset_options}{extra_options}""" else: - cmd = "cd '" + os.path.join(env['RUN_DIR'], "python") + "' && " + env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " +\ - "--backend " + env['MLC_MLPERF_BACKEND'] + " --scenario=" + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + \ - mode_extra_options + dataset_options + extra_options + cmd = f"""cd {x}{os.path.join(env['RUN_DIR'], 'python')}{x} && {x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py --backend {env['MLC_MLPERF_BACKEND']} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']}{env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']}{scenario_extra_options}{mode_extra_options}{dataset_options}{extra_options}""" + env['SKIP_VERIFY_ACCURACY'] = True elif "bert" in env['MLC_MODEL']: @@ -311,13 +296,11 @@ def get_run_cmd_reference( quantization_options = " --quantized" else: quantization_options = "" - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " run.py --backend=" + env['MLC_MLPERF_BACKEND'] + " --scenario=" + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + \ - mode_extra_options + dataset_options + quantization_options + + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} run.py --backend={env['MLC_MLPERF_BACKEND']} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']}{env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']}{scenario_extra_options}{mode_extra_options}{dataset_options}{quantization_options}""" + if env['MLC_MLPERF_BACKEND'] == "deepsparse": - cmd += " --batch_size=" + \ - env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') + \ - " --model_path=" + env['MODEL_FILE'] + cmd += f""" --batch_size={env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '1')} --model_path={x}{env['MODEL_FILE']}{x}""" if env.get('MLC_MLPERF_CUSTOM_MODEL_PATH', '') != '': env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MODEL_FILE'] @@ -331,15 +314,15 @@ def get_run_cmd_reference( elif "rnnt" in env['MLC_MODEL']: env['RUN_DIR'] = env['MLC_MLPERF_INFERENCE_RNNT_PATH'] - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " run.py --backend " + env['MLC_MLPERF_BACKEND'] + \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --manifest " + env['MLC_DATASET_PREPROCESSED_JSON'] + \ - " --dataset_dir " + os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], "..") + \ - " --pytorch_config_toml " + os.path.join("pytorch", "configs", "rnnt.toml") + \ - " --pytorch_checkpoint " + env['MLC_ML_MODEL_FILE_WITH_PATH'] + \ - " --log_dir " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + dataset_options + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} run.py --backend {env['MLC_MLPERF_BACKEND']} \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --manifest {x}{env['MLC_DATASET_PREPROCESSED_JSON']}{x} \ + --dataset_dir {x}{os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], '..')}{x} \ + --pytorch_config_toml {x}{os.path.join('pytorch', 'configs', 'rnnt.toml')}{x} \ + --pytorch_checkpoint {x}{env['MLC_ML_MODEL_FILE_WITH_PATH']}{x} \ + --log_dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" + env['SKIP_VERIFY_ACCURACY'] = True elif "stable-diffusion-xl" in env['MLC_MODEL']: @@ -356,21 +339,24 @@ def get_run_cmd_reference( backend = env['MLC_MLPERF_BACKEND'] max_batchsize = env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --profile " + 'stable-diffusion-xl-pytorch ' + \ - " --dataset " + 'coco-1024' + \ - " --dataset-path " + env['MLC_DATASET_PATH_ROOT'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'].replace("bfloat", "bf").replace("float", "fp") + \ - " --device " + device + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + \ - " --output " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - " --model-path " + env['MLC_ML_MODEL_PATH'] + + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --profile stable-diffusion-xl-pytorch \ + --dataset coco-1024 \ + --dataset-path {x}{env['MLC_DATASET_PATH_ROOT']}{x} \ + --dtype {env['MLC_MLPERF_MODEL_PRECISION'].replace("bfloat", "bf").replace("float", "fp")} \ + --device {device} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + {scenario_extra_options} {mode_extra_options} \ + --output {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --model-path {x}{env['MLC_ML_MODEL_PATH']}{x}""" + if "--max-batchsize" not in cmd: - cmd += " --max-batchsize " + max_batchsize + cmd += f" --max-batchsize {max_batchsize}" + if env.get('MLC_COCO2014_SAMPLE_ID_PATH', '') != '': - cmd += " --ids-path " + env['MLC_COCO2014_SAMPLE_ID_PATH'] + cmd += f""" --ids-path {x}{env['MLC_COCO2014_SAMPLE_ID_PATH']}{x}""" elif "llama2-70b" in env['MLC_MODEL']: env['RUN_DIR'] = os.path.join( @@ -379,14 +365,14 @@ def get_run_cmd_reference( "llama2-70b") backend = env['MLC_MLPERF_BACKEND'] - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['MLC_DATASET_PREPROCESSED_PATH'] + \ - " --device " + device.replace("cuda", "cuda:0") + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + \ - " --output-log-dir " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'] + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --dataset-path {x}{env['MLC_DATASET_PREPROCESSED_PATH']}{x} \ + --device {device.replace("cuda", "cuda:0")} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + {scenario_extra_options} {mode_extra_options} \ + --output-log-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --dtype {env['MLC_MLPERF_MODEL_PRECISION']}""" if env.get('MLC_MLPERF_INFERENCE_API_SERVER', '') != '': env['MLC_VLLM_SERVER_MODEL_NAME'] = env.get( @@ -396,7 +382,7 @@ def get_run_cmd_reference( --model-path {env['MLC_VLLM_SERVER_MODEL_NAME']} \ --api-model-name {env['MLC_VLLM_SERVER_MODEL_NAME']} --vllm """ else: - cmd += f" --model-path {env['LLAMA2_CHECKPOINT_PATH']}" + cmd += f""" --model-path {x}{env['LLAMA2_CHECKPOINT_PATH']}{x}""" if env.get('MLC_MLPERF_INFERENCE_NUM_WORKERS', '') != '': cmd += f" --num-workers {env['MLC_MLPERF_INFERENCE_NUM_WORKERS']}" @@ -410,15 +396,16 @@ def get_run_cmd_reference( "language", "mixtral-8x7b") backend = env['MLC_MLPERF_BACKEND'] - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] + \ - " --device " + device.replace("cuda", "cuda:0") + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + \ - " --output-log-dir " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'] + \ - " --model-path " + env['MIXTRAL_CHECKPOINT_PATH'] + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --dataset-path {x}{env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH']}{x} \ + --device {device.replace('cuda', 'cuda:0')} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + {scenario_extra_options} {mode_extra_options} \ + --output-log-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --dtype {env['MLC_MLPERF_MODEL_PRECISION']} \ + --model-path {x}{env['MIXTRAL_CHECKPOINT_PATH']}{x}""" + cmd = cmd.replace("--count", "--total-sample-count") cmd = cmd.replace("--max-batchsize", "--batch-size") @@ -426,11 +413,11 @@ def get_run_cmd_reference( env['RUN_DIR'] = env['MLC_MLPERF_INFERENCE_3DUNET_PATH'] backend = env['MLC_MLPERF_BACKEND'] if env['MLC_MLPERF_BACKEND'] != 'tf' else 'tensorflow' - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " run.py --backend=" + backend + " --scenario=" + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - " --model=" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + \ - " --preprocessed_data_dir=" + env['MLC_DATASET_KITS19_PREPROCESSED_PATH'] + \ - scenario_extra_options + mode_extra_options + dataset_options + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} run.py --backend={backend} --scenario={env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + --model={x}{env['MLC_ML_MODEL_FILE_WITH_PATH']}{x} \ + --preprocessed_data_dir={x}{env['MLC_DATASET_KITS19_PREPROCESSED_PATH']}{x} \ + {scenario_extra_options} {mode_extra_options} {dataset_options}""" env['LOG_PATH'] = env['MLC_MLPERF_OUTPUT_DIR'] env['SKIP_VERIFY_ACCURACY'] = True @@ -465,12 +452,11 @@ def get_run_cmd_reference( if env['MLC_MLPERF_LOADGEN_MODE'] == "accuracy" and env['MLC_MLPERF_LOADGEN_SCENARIO'] == "Offline": mode_extra_options += " --samples-per-query-offline=1" - cmd = " ./run_local.sh " + env['MLC_MLPERF_BACKEND'] + \ - ' dlrm ' + dataset + ' ' + env['MLC_MLPERF_DEVICE'] + " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - config + mlperf_bin_loader_string + \ - ' --samples-to-aggregate-quantile-file=./tools/dist_quantile.txt ' + \ - scenario_extra_options + mode_extra_options + dataset_options + gpu_options + cmd = f""" ./run_local.sh {env['MLC_MLPERF_BACKEND']} dlrm {dataset} {env['MLC_MLPERF_DEVICE']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + {config} {mlperf_bin_loader_string} \ + --samples-to-aggregate-quantile-file=./tools/dist_quantile.txt {scenario_extra_options} {mode_extra_options} {dataset_options} {gpu_options}""" + cmd = cmd.replace("--count", "--count-queries") env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] @@ -489,15 +475,16 @@ def get_run_cmd_reference( mode_extra_options += " --dataset igbh-dgl-tiny --profile debug-dgl " # have to add the condition for running in debug mode or real run mode - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['MLC_DATASET_IGBH_PATH'] + \ - " --device " + device.replace("cuda", "gpu") + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + \ - " --output " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - ' --dtype ' + dtype_rgat + \ - " --model-path " + env['RGAT_CHECKPOINT_PATH'] + + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --dataset-path {x}{env['MLC_DATASET_IGBH_PATH']}{x} \ + --device {device.replace("cuda", "gpu")} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ + {scenario_extra_options} {mode_extra_options} \ + --output {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --dtype {dtype_rgat} \ + --model-path {x}{env['RGAT_CHECKPOINT_PATH']}{x}""" if env.get('MLC_ACTIVATE_RGAT_IN_MEMORY', '') == "yes": cmd += " --in-memory " @@ -510,15 +497,15 @@ def get_run_cmd_reference( if int(env.get('MLC_MLPERF_INFERENCE_TP_SIZE', '')) > 1: env['VLLM_WORKER_MULTIPROC_METHOD'] = "spawn" - - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['MLC_DATASET_LLAMA3_PATH'] + \ - " --output-log-dir " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - ' --dtype ' + env['MLC_MLPERF_MODEL_PRECISION'] + \ - " --model-path " + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + \ - " --tensor-parallel-size " + env['MLC_MLPERF_INFERENCE_TP_SIZE'] + \ - " --vllm " + + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --dataset-path {x}{env['MLC_DATASET_LLAMA3_PATH']}{x} \ + --output-log-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --dtype {env['MLC_MLPERF_MODEL_PRECISION']} \ + --model-path {x}{env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH']}{x} \ + --tensor-parallel-size {env['MLC_MLPERF_INFERENCE_TP_SIZE']} \ + --vllm""" if env.get('MLC_MLPERF_INFERENCE_NUM_WORKERS', '') != '': cmd += f" --num-workers {env['MLC_MLPERF_INFERENCE_NUM_WORKERS']}" @@ -532,16 +519,16 @@ def get_run_cmd_reference( "automotive", "3d-object-detection") - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \ - " --dataset waymo" + \ - " --dataset-path " + env['MLC_DATASET_WAYMO_PATH'] + \ - " --lidar-path " + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] + \ - " --segmentor-path " + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] + \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \ - " --output " + env['MLC_MLPERF_OUTPUT_DIR'] + \ - " --dtype " + env['MLC_MLPERF_MODEL_PRECISION'].replace("float", "fp") + \ - scenario_extra_options + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + mode_extra_options + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ + --dataset waymo \ + --dataset-path {x}{env['MLC_DATASET_WAYMO_PATH']}{x} \ + --lidar-path {x}{env['MLC_ML_MODEL_POINT_PAINTING_PATH']}{x} \ + --segmentor-path {x}{env['MLC_ML_MODEL_DPLAB_RESNET50_PATH']}{x} \ + --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ + --output {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ + --dtype {env['MLC_MLPERF_MODEL_PRECISION'].replace('float', 'fp')} \ + {scenario_extra_options} \ + {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {mode_extra_options}""" if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '': cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}" diff --git a/script/benchmark-program/run.bat b/script/benchmark-program/run.bat index ccc797361..369ad5fe7 100644 --- a/script/benchmark-program/run.bat +++ b/script/benchmark-program/run.bat @@ -5,7 +5,7 @@ if "%MLC_RUN_DIR%" == "" ( exit 1 ) -cd %MLC_RUN_DIR% +cd "%MLC_RUN_DIR%" if "%MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM%" == "True" ( echo ***************************************************** diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 1188ff8e8..0fa1e2616 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -5,6 +5,8 @@ def preprocess(i): os_info = i['os_info'] + q = '"' if os_info['platform'] == 'windows' else "'" + env = i['env'] CPPFLAGS = env.get('+ CPPFLAGS', []) env['MLC_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) @@ -13,14 +15,13 @@ def preprocess(i): env['MLC_F_COMPILER_FLAGS'] = " ".join(env.get('+ FFLAGS', [])) CPATH = env.get('+CPATH', []) - env['MLC_C_INCLUDE_PATH'] = " -I".join([" "] + - env.get('+C_INCLUDE_PATH', []) + - CPATH) - env['MLC_CPLUS_INCLUDE_PATH'] = " -I".join( - [" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) - env['MLC_F_INCLUDE_PATH'] = " -I".join([" "] + - env.get('+F_INCLUDE_PATH', []) + - CPATH) + env['MLC_C_INCLUDE_PATH'] = " -I".join([f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH]) + + env['MLC_C_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH) + + env['MLC_CPLUS_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) + + env['MLC_F_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+F_INCLUDE_PATH', []) + CPATH) # If windows, need to extend it more ... if os_info['platform'] == 'windows' and env.get( @@ -53,8 +54,8 @@ def preprocess(i): env['MLC_LINKER_COMPILE_FLAGS'] = env['MLC_F_COMPILER_FLAGS'] env['MLC_LINKER_FLAGS'] = env['MLC_F_LINKER_FLAGS'] - env['MLC_LD_LIBRARY_PATH'] = " -L".join([" "] + - env.get('+LD_LIBRARY_PATH', [])) + env['MLC_LD_LIBRARY_PATH'] = " -L".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+LD_LIBRARY_PATH', [])) + env['MLC_SOURCE_FOLDER_PATH'] = env['MLC_SOURCE_FOLDER_PATH'] if 'MLC_SOURCE_FOLDER_PATH' in env else env[ 'MLC_TMP_CURRENT_SCRIPT_PATH'] if 'MLC_TMP_CURRENT_SCRIPT_PATH' in env else '' From 5825ecd2af91b75d11bdeb7746adf918e03f3bfd Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:35:16 +0530 Subject: [PATCH 25/55] test commit --- script/extract-file/run.bat | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/extract-file/run.bat b/script/extract-file/run.bat index bfc066533..a58a0ad9e 100644 --- a/script/extract-file/run.bat +++ b/script/extract-file/run.bat @@ -20,20 +20,20 @@ if exist "%MLC_EXTRACT_EXTRACTED_FILENAME%" ( ) if "!require_extract!" == "1" ( - if not "%MLC_EXTRACT_CMD0%" == "" ( + if not %MLC_EXTRACT_CMD0% == "" ( echo. echo %MLC_EXTRACT_CMD0% - cmd /c "%MLC_EXTRACT_CMD0%" + cmd /c %MLC_EXTRACT_CMD0% IF !ERRORLEVEL! NEQ 0 EXIT 1 ) echo. echo %MLC_EXTRACT_CMD% - cmd /c "%MLC_EXTRACT_CMD%" + cmd /c %MLC_EXTRACT_CMD% IF !ERRORLEVEL! NEQ 0 EXIT 1 echo. echo %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% - cmd /c "%MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD%" + cmd /c %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% IF !ERRORLEVEL! NEQ 0 EXIT 1 ) From a54971b3ee09af6ca1b9e8a71deed158152e6968 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Apr 2025 11:05:38 +0000 Subject: [PATCH 26/55] [Automated Commit] Format Codebase [skip ci] --- .../customize.py | 2 +- script/compile-program/customize.py | 35 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 2cedc7d9c..d0eef3b51 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -497,7 +497,7 @@ def get_run_cmd_reference( if int(env.get('MLC_MLPERF_INFERENCE_TP_SIZE', '')) > 1: env['VLLM_WORKER_MULTIPROC_METHOD'] = "spawn" - + cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \ --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \ --dataset-path {x}{env['MLC_DATASET_LLAMA3_PATH']}{x} \ diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 0fa1e2616..ae59862ba 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -15,13 +15,29 @@ def preprocess(i): env['MLC_F_COMPILER_FLAGS'] = " ".join(env.get('+ FFLAGS', [])) CPATH = env.get('+CPATH', []) - env['MLC_C_INCLUDE_PATH'] = " -I".join([f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH]) - - env['MLC_C_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH) - - env['MLC_CPLUS_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) - - env['MLC_F_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+F_INCLUDE_PATH', []) + CPATH) + env['MLC_C_INCLUDE_PATH'] = " -I".join([f"""{q}{path}{q}""" for path in [ + " "] + env.get('+C_INCLUDE_PATH', []) + CPATH]) + + env['MLC_C_INCLUDE_PATH'] = " -I".join( + f"""{q}{path}{q}""" for path in [" "] + + env.get( + '+C_INCLUDE_PATH', + []) + + CPATH) + + env['MLC_CPLUS_INCLUDE_PATH'] = " -I".join( + f"""{q}{path}{q}""" for path in [" "] + + env.get( + '+CPLUS_INCLUDE_PATH', + []) + + CPATH) + + env['MLC_F_INCLUDE_PATH'] = " -I".join( + f"""{q}{path}{q}""" for path in [" "] + + env.get( + '+F_INCLUDE_PATH', + []) + + CPATH) # If windows, need to extend it more ... if os_info['platform'] == 'windows' and env.get( @@ -54,8 +70,9 @@ def preprocess(i): env['MLC_LINKER_COMPILE_FLAGS'] = env['MLC_F_COMPILER_FLAGS'] env['MLC_LINKER_FLAGS'] = env['MLC_F_LINKER_FLAGS'] - env['MLC_LD_LIBRARY_PATH'] = " -L".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+LD_LIBRARY_PATH', [])) - + env['MLC_LD_LIBRARY_PATH'] = " -L".join(f"""{q}{path}{q}""" for path in [ + " "] + env.get('+LD_LIBRARY_PATH', [])) + env['MLC_SOURCE_FOLDER_PATH'] = env['MLC_SOURCE_FOLDER_PATH'] if 'MLC_SOURCE_FOLDER_PATH' in env else env[ 'MLC_TMP_CURRENT_SCRIPT_PATH'] if 'MLC_TMP_CURRENT_SCRIPT_PATH' in env else '' From e42f64353103866fd1c8c1fa1d5620a884ffbef5 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:51:18 +0530 Subject: [PATCH 27/55] fix link issue --- script/compile-program/customize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 0fa1e2616..33410e574 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -15,13 +15,13 @@ def preprocess(i): env['MLC_F_COMPILER_FLAGS'] = " ".join(env.get('+ FFLAGS', [])) CPATH = env.get('+CPATH', []) - env['MLC_C_INCLUDE_PATH'] = " -I".join([f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH]) + env['MLC_C_INCLUDE_PATH'] = " ".join([f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH]) - env['MLC_C_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+C_INCLUDE_PATH', []) + CPATH) + env['MLC_C_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH) - env['MLC_CPLUS_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) + env['MLC_CPLUS_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) - env['MLC_F_INCLUDE_PATH'] = " -I".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+F_INCLUDE_PATH', []) + CPATH) + env['MLC_F_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+F_INCLUDE_PATH', []) + CPATH) # If windows, need to extend it more ... if os_info['platform'] == 'windows' and env.get( @@ -54,8 +54,8 @@ def preprocess(i): env['MLC_LINKER_COMPILE_FLAGS'] = env['MLC_F_COMPILER_FLAGS'] env['MLC_LINKER_FLAGS'] = env['MLC_F_LINKER_FLAGS'] - env['MLC_LD_LIBRARY_PATH'] = " -L".join(f"""{q}{path}{q}""" for path in [" "] + env.get('+LD_LIBRARY_PATH', [])) - + env['MLC_LD_LIBRARY_PATH'] = " ".join(f"""-L{q}{path}{q}""" for path in env.get('+LD_LIBRARY_PATH', [])) + env['MLC_SOURCE_FOLDER_PATH'] = env['MLC_SOURCE_FOLDER_PATH'] if 'MLC_SOURCE_FOLDER_PATH' in env else env[ 'MLC_TMP_CURRENT_SCRIPT_PATH'] if 'MLC_TMP_CURRENT_SCRIPT_PATH' in env else '' From 9fdc70117bb61bf85b00add6a7677d5f508a3537 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Apr 2025 11:22:57 +0000 Subject: [PATCH 28/55] [Automated Commit] Format Codebase [skip ci] --- script/compile-program/customize.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 33410e574..8a4a7a669 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -15,13 +15,17 @@ def preprocess(i): env['MLC_F_COMPILER_FLAGS'] = " ".join(env.get('+ FFLAGS', [])) CPATH = env.get('+CPATH', []) - env['MLC_C_INCLUDE_PATH'] = " ".join([f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH]) + env['MLC_C_INCLUDE_PATH'] = " ".join( + [f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH]) - env['MLC_C_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH) + env['MLC_C_INCLUDE_PATH'] = " ".join( + f"""-I{q}{path}{q}""" for path in env.get('+C_INCLUDE_PATH', []) + CPATH) - env['MLC_CPLUS_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) + env['MLC_CPLUS_INCLUDE_PATH'] = " ".join( + f"""-I{q}{path}{q}""" for path in env.get('+CPLUS_INCLUDE_PATH', []) + CPATH) - env['MLC_F_INCLUDE_PATH'] = " ".join(f"""-I{q}{path}{q}""" for path in env.get('+F_INCLUDE_PATH', []) + CPATH) + env['MLC_F_INCLUDE_PATH'] = " ".join( + f"""-I{q}{path}{q}""" for path in env.get('+F_INCLUDE_PATH', []) + CPATH) # If windows, need to extend it more ... if os_info['platform'] == 'windows' and env.get( @@ -54,7 +58,8 @@ def preprocess(i): env['MLC_LINKER_COMPILE_FLAGS'] = env['MLC_F_COMPILER_FLAGS'] env['MLC_LINKER_FLAGS'] = env['MLC_F_LINKER_FLAGS'] - env['MLC_LD_LIBRARY_PATH'] = " ".join(f"""-L{q}{path}{q}""" for path in env.get('+LD_LIBRARY_PATH', [])) + env['MLC_LD_LIBRARY_PATH'] = " ".join( + f"""-L{q}{path}{q}""" for path in env.get('+LD_LIBRARY_PATH', [])) env['MLC_SOURCE_FOLDER_PATH'] = env['MLC_SOURCE_FOLDER_PATH'] if 'MLC_SOURCE_FOLDER_PATH' in env else env[ 'MLC_TMP_CURRENT_SCRIPT_PATH'] if 'MLC_TMP_CURRENT_SCRIPT_PATH' in env else '' From daa142bbc49cbfd03e4f3f3b2074afd6248e3a09 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:53:32 +0530 Subject: [PATCH 29/55] update customize.py --- script/compile-program/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 33410e574..cf04d09db 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -61,7 +61,6 @@ def preprocess(i): return {'return': 0} - def postprocess(i): return {'return': 0} From f5ac95928bdc542b27375736e1d19a08c877ad19 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Apr 2025 11:24:11 +0000 Subject: [PATCH 30/55] [Automated Commit] Format Codebase [skip ci] --- script/compile-program/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 4e962aeb4..8a4a7a669 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -66,6 +66,7 @@ def preprocess(i): return {'return': 0} + def postprocess(i): return {'return': 0} From 5720689aac3f0413c3f1f4753880e03bee13683d Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:56:54 +0530 Subject: [PATCH 31/55] update customize.py --- script/compile-program/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 8a4a7a669..e85349da1 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -8,6 +8,7 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" env = i['env'] + CPPFLAGS = env.get('+ CPPFLAGS', []) env['MLC_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) env['MLC_CXX_COMPILER_FLAGS'] = " ".join( From 1bc4b05fc48a0aa24419a2b3286180926e0e1591 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Apr 2025 11:27:09 +0000 Subject: [PATCH 32/55] [Automated Commit] Format Codebase [skip ci] --- script/compile-program/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index e85349da1..7eb07cbc4 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -8,7 +8,7 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" env = i['env'] - + CPPFLAGS = env.get('+ CPPFLAGS', []) env['MLC_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) env['MLC_CXX_COMPILER_FLAGS'] = " ".join( From 3d95aa48154527f400bfa747fd7e430c3e5e7960 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 16:57:41 +0530 Subject: [PATCH 33/55] update customize.py --- script/compile-program/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 7eb07cbc4..8a4a7a669 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -8,7 +8,6 @@ def preprocess(i): q = '"' if os_info['platform'] == 'windows' else "'" env = i['env'] - CPPFLAGS = env.get('+ CPPFLAGS', []) env['MLC_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) env['MLC_CXX_COMPILER_FLAGS'] = " ".join( From f5928e0ca09eaa4e599362c8b986cf03b6d83476 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 17:08:02 +0530 Subject: [PATCH 34/55] test commit - handle expansion at runtime --- script/extract-file/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/extract-file/run.bat b/script/extract-file/run.bat index a58a0ad9e..defc40eab 100644 --- a/script/extract-file/run.bat +++ b/script/extract-file/run.bat @@ -20,7 +20,7 @@ if exist "%MLC_EXTRACT_EXTRACTED_FILENAME%" ( ) if "!require_extract!" == "1" ( - if not %MLC_EXTRACT_CMD0% == "" ( + if not "!MLC_EXTRACT_CMD0!" == "" ( echo. echo %MLC_EXTRACT_CMD0% cmd /c %MLC_EXTRACT_CMD0% From 3e460fcea38f11efdcebba750a61621ef2d40005 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 17:32:40 +0530 Subject: [PATCH 35/55] fix for path issue --- script/benchmark-program/run-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/benchmark-program/run-ubuntu.sh b/script/benchmark-program/run-ubuntu.sh index dfca75282..9771497b3 100644 --- a/script/benchmark-program/run-ubuntu.sh +++ b/script/benchmark-program/run-ubuntu.sh @@ -2,7 +2,7 @@ MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} -cd ${MLC_TMP_CURRENT_SCRIPT_PATH} +cd "${MLC_TMP_CURRENT_SCRIPT_PATH}" if [ ${MLC_ENABLE_NUMACTL} == "1" ]; then sudo apt-get install numactl fi From 0cf6e9014f1c5a5ae4b00240adf08f0eaeb79193 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 17:54:01 +0530 Subject: [PATCH 36/55] fix for space in path --- script/benchmark-program/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/benchmark-program/run.sh b/script/benchmark-program/run.sh index 011e6a8fe..6caace406 100755 --- a/script/benchmark-program/run.sh +++ b/script/benchmark-program/run.sh @@ -20,12 +20,12 @@ if [[ ${MLC_MLPERF_POWER} == "yes" && ${MLC_MLPERF_LOADGEN_MODE} == "performance fi # Run -if [ -z ${MLC_RUN_DIR} ]; then +if [ -z "${MLC_RUN_DIR}" ]; then echo "MLC_RUN_DIR is not set" exit 1 fi -cd ${MLC_RUN_DIR} +cd "${MLC_RUN_DIR}" if [[ "${MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM}" == "True" ]]; then echo "*****************************************************" From 00c491ceac86a813536567285f272ec561400393 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 19:21:12 +0530 Subject: [PATCH 37/55] fixes the output path when there is space - compiler linkage --- script/compile-program/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/compile-program/run.sh b/script/compile-program/run.sh index 171b99070..c13c1cef3 100644 --- a/script/compile-program/run.sh +++ b/script/compile-program/run.sh @@ -65,7 +65,7 @@ done echo "" echo "Linking ..." echo "" -CMD="${MLC_LINKER_WITH_PATH} ${MLC_LINKER_COMPILE_FLAGS} *.o -o ${RUN_DIR}/${BIN_NAME} ${MLC_LD_LIBRARY_PATH} ${MLC_LINKER_FLAGS}" +CMD="${MLC_LINKER_WITH_PATH} ${MLC_LINKER_COMPILE_FLAGS} *.o -o \"${RUN_DIR}/${BIN_NAME}\" ${MLC_LD_LIBRARY_PATH} ${MLC_LINKER_FLAGS}" echo $CMD eval $CMD From 2b7c8fc7502b10d4326b7ca0f7dc66c3ca17a2dd Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 21:40:34 +0530 Subject: [PATCH 38/55] fix space in path issue for dump freeze --- script/dump-pip-freeze/run.bat | 2 +- script/dump-pip-freeze/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/dump-pip-freeze/run.bat b/script/dump-pip-freeze/run.bat index 18f6b56e5..48224d0e3 100644 --- a/script/dump-pip-freeze/run.bat +++ b/script/dump-pip-freeze/run.bat @@ -1,4 +1,4 @@ if not "%MLC_FAKE_RUN%" == "yes" ( - %MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\dump.py + "%MLC_PYTHON_BIN_WITH_PATH%" "%MLC_TMP_CURRENT_SCRIPT_PATH%\dump.py" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% ) diff --git a/script/dump-pip-freeze/run.sh b/script/dump-pip-freeze/run.sh index 8d4d76e1a..0e379ea14 100644 --- a/script/dump-pip-freeze/run.sh +++ b/script/dump-pip-freeze/run.sh @@ -25,4 +25,4 @@ function run() { #Add your run commands here... # run "$MLC_RUN_CMD" -run "${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/dump.py" +run "\"${MLC_PYTHON_BIN_WITH_PATH}\" \"${MLC_TMP_CURRENT_SCRIPT_PATH}/dump.py\"" From b063ffa23615bc9ba9afbfdeb98883d328798f89 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 22:42:31 +0530 Subject: [PATCH 39/55] run benchmark with forked inference repo --- .../test-mlperf-inference-resnet50-closed-division.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index abc923e8e..77624487a 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -84,12 +84,12 @@ jobs: - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.anandhu-eng@inference,_branch.patch-34 - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.anandhu-eng@inference,_branch.patch-34 # Step for Linux/MacOS - name: Randomly Execute Step (Linux/MacOS) From f4bda4f67a3d09988ef6780aa2640ae2260904e5 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 23:01:14 +0530 Subject: [PATCH 40/55] corrected git repo link --- .../test-mlperf-inference-resnet50-closed-division.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 77624487a..663dff628 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -84,12 +84,12 @@ jobs: - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.anandhu-eng@inference,_branch.patch-34 + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference,_branch.patch-34 - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.anandhu-eng@inference,_branch.patch-34 + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --adr.inference-src.tags=_repo.https://github.com/anandhu-eng/inference,_branch.patch-34 # Step for Linux/MacOS - name: Randomly Execute Step (Linux/MacOS) From c5ec48abc86e5fe87e4a4dad35b0b21771959b6e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 23:09:44 +0530 Subject: [PATCH 41/55] fix for space in path --- script/get-generic-python-lib/run.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-generic-python-lib/run.bat b/script/get-generic-python-lib/run.bat index af803d720..65f281b0c 100644 --- a/script/get-generic-python-lib/run.bat +++ b/script/get-generic-python-lib/run.bat @@ -1,4 +1,4 @@ -IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET "MLC_TMP_CURRENT_SCRIPT_PATH=%CD%" +IF NOT DEFINED MLC_TMP_CURRENT_SCRIPT_PATH SET MLC_TMP_CURRENT_SCRIPT_PATH=%CD% -%MLC_PYTHON_BIN_WITH_PATH% "%MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py" +"%MLC_PYTHON_BIN_WITH_PATH%" "%MLC_TMP_CURRENT_SCRIPT_PATH%\detect-version.py" IF %ERRORLEVEL% NEQ 0 EXIT 1 From 8deaeb0d91dbd40b1aee4e18ec0353eb76c76aa8 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 24 Apr 2025 05:25:03 +0530 Subject: [PATCH 42/55] fix for paths --- script/app-mlperf-inference/verify_accuracy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/verify_accuracy.sh b/script/app-mlperf-inference/verify_accuracy.sh index 5a8cec92a..899d33ca2 100644 --- a/script/app-mlperf-inference/verify_accuracy.sh +++ b/script/app-mlperf-inference/verify_accuracy.sh @@ -1,4 +1,4 @@ #/bin/bash echo "Running: $CMD" -eval $CMD +eval "$CMD" test $? -eq 0 || exit $? From 61ebea34ec2fdff92e374d87f971d1fe677b61ee Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 11:49:33 +0100 Subject: [PATCH 43/55] Update test-mlperf-inference-rgat.yml --- .../workflows/test-mlperf-inference-rgat.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 948b5e802..e04023993 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -9,6 +9,32 @@ on: - '!**.md' jobs: + fetch-secret: + runs-on: ubuntu-latest + outputs: + encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} + steps: + - name: Load secret + id: op-load-secret + uses: 1password/load-secrets-action@v2 + with: + export-env: false + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential + + - name: Encrypt secret + id: encrypt-secret + env: + ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} + run: | + # AES-256 encrypt + encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ + openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ + -pass pass:"$ENCRYPTION_KEY" -base64 -A) + + echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT + rgat-inference-run: name: ${{ matrix.os }} - ${{ matrix.backend }} - ${{ matrix.implementation }} needs: [fetch-secret] From 0279a9c4195b7d7a338de6c5932d18f67626600c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 11:54:11 +0100 Subject: [PATCH 44/55] Update test-mlperf-inference-mlcommons-cpp-resnet50.yml --- .../test-mlperf-inference-mlcommons-cpp-resnet50.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index db5e4d719..59ef633ee 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -43,11 +43,9 @@ jobs: fail-fast: false matrix: python-version: [ "3.12" ] - llvm-version: [ "15.0.6", "16.0.4", "17.0.6" ] + compiler-string: [ "--adr.compiler.tags=gcc", "--adr.compiler.tags=aocc", "--adr.compiler.tags=llvm --adr.compiler.version=17.0.6" ] os: [ubuntu-latest, windows-latest, macos-latest] exclude: - - llvm-version: "15.0.6" - - llvm-version: "16.0.4" - os: windows-latest - os: macos-latest @@ -63,8 +61,7 @@ jobs: - name: Pull MLOps repository run: | mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - mlcr --quiet --tags=get,sys-utils-cm - mlcr --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }} + mlcr --quiet --tags=get,sys-utils-mlc - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} if: matrix.os == 'windows-latest' run: | @@ -72,7 +69,7 @@ jobs: - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} if: matrix.os != 'windows-latest' run: | - mlcr app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet + mlcr app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet ${{ matrix.compiler-string }} - name: Randomly Execute Step id: random-check run: | From 0fa0e8551927a2fe4a6d0f504fe1afcce8791884 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 13:29:34 +0100 Subject: [PATCH 45/55] Update test-amd-mlperf-inference-implementations.yml --- .github/workflows/test-amd-mlperf-inference-implementations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-amd-mlperf-inference-implementations.yml b/.github/workflows/test-amd-mlperf-inference-implementations.yml index bc01ad20c..5cb888fef 100644 --- a/.github/workflows/test-amd-mlperf-inference-implementations.yml +++ b/.github/workflows/test-amd-mlperf-inference-implementations.yml @@ -13,6 +13,7 @@ jobs: matrix: python-version: [ "3.12" ] model: [ "llama2-70b-99.9" ] + steps: - name: Test MLPerf Inference AMD (build only) ${{ matrix.model }} run: | From ec70c455c1e835c7087037bcc8172dde6a2cb3c1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 13:33:57 +0100 Subject: [PATCH 46/55] Update test-mlperf-inference-mlcommons-cpp-resnet50.yml --- .../workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 59ef633ee..147fb0ff4 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.12" ] - compiler-string: [ "--adr.compiler.tags=gcc", "--adr.compiler.tags=aocc", "--adr.compiler.tags=llvm --adr.compiler.version=17.0.6" ] + compiler-string: [ "--adr.compiler.tags=gcc", "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes", "--adr.compiler.tags=llvm --adr.compiler.version=17.0.6" ] os: [ubuntu-latest, windows-latest, macos-latest] exclude: - os: windows-latest From 53162a1c07e8104dbe64aa951b567b88b317f6d1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 13:38:59 +0100 Subject: [PATCH 47/55] Update run-tests-on-modified-meta.yml --- .github/workflows/run-tests-on-modified-meta.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 7304a42dd..244b18a6f 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -39,6 +39,7 @@ jobs: process_modified_files: runs-on: ubuntu-latest needs: get_modified_files + if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != '' strategy: fail-fast: false matrix: From 93a986ca7bc32f03ba0fed870d5537fc00f6ed40 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 13:56:30 +0100 Subject: [PATCH 48/55] Update test-mlperf-inference-rgat.yml --- .../workflows/test-mlperf-inference-rgat.yml | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index e04023993..2391c65fc 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -9,35 +9,9 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT rgat-inference-run: name: ${{ matrix.os }} - ${{ matrix.backend }} - ${{ matrix.implementation }} - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -64,24 +38,3 @@ jobs: run: | mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - - name: Load secret - if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Push Results - env: - GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }} - run: | - git config --global user.name "mlcommons-bot" - git config --global user.email "mlcommons-bot@users.noreply.github.com" - git config --global credential.https://github.com.helper "" - git config --global credential.https://github.com.helper "!gh auth git-credential" - git config --global credential.https://gist.github.com.helper "" - git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet From 4e3caadc50ae9b59ee6e956d28fc4c4d97cd1c6a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 14:15:15 +0100 Subject: [PATCH 49/55] Update test-mlperf-inference-retinanet.yml --- .../test-mlperf-inference-retinanet.yml | 84 +------------------ 1 file changed, 4 insertions(+), 80 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 93cec255c..860f23fe6 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -9,34 +9,8 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT mlc-run: - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -45,6 +19,7 @@ jobs: python-version: [ "3.12" ] backend: [ "onnxruntime", "pytorch" ] implementation: [ "python", "cpp" ] + compiler-string: [ "", "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes" ] exclude: - backend: pytorch implementation: cpp @@ -52,6 +27,8 @@ jobs: implementation: cpp - os: macos-latest implementation: cpp + - implementation: python + compiler-string: "--adr.compiler.tags=aocc --env.MLC_AOCC_ACCEPT_EULA=yes" steps: - uses: actions/checkout@v3 @@ -78,57 +55,4 @@ jobs: - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }} if: matrix.os != 'windows-latest' run: | - mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1 - - # Step for Linux/MacOS - - name: Randomly Execute Step (Linux/MacOS) - if: runner.os != 'Windows' - run: | - RANDOM_NUMBER=$((RANDOM % 10)) - echo "Random number is $RANDOM_NUMBER" - if [ "$RANDOM_NUMBER" -eq 0 ]; then - echo "run_step=true" >> $GITHUB_ENV - else - echo "run_step=false" >> $GITHUB_ENV - fi - - # Step for Windows - - name: Randomly Execute Step (Windows) - if: runner.os == 'Windows' - run: | - $RANDOM_NUMBER = Get-Random -Maximum 10 - Write-Host "Random number is $RANDOM_NUMBER" - if ($RANDOM_NUMBER -eq 0) { - Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append - } else { - Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append - } - - - name: Decrypt secret - id: decrypt-secret - shell: bash - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" - - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT - - name: Push Results - env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} - if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - run: | - git config --global user.name "mlcommons-bot" - git config --global user.email "mlcommons-bot@users.noreply.github.com" - git config --global credential.https://github.com.helper "" - git config --global credential.https://github.com.helper "!gh auth git-credential" - git config --global credential.https://gist.github.com.helper "" - git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet + mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1 ${{ matrix.compiler-string }} From 3c7829b1ad8f9d17a3535141a5d46201f40a97da Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 24 Apr 2025 19:58:06 +0530 Subject: [PATCH 50/55] Replace print with MLC Logger (#396) * replace print with logger --- .../customize.py | 2 +- .../app-image-corner-detection/customize.py | 3 +- .../app-loadgen-generic-python/customize.py | 8 ++- .../app-mlperf-inference-dummy/customize.py | 6 +- .../app-mlperf-inference-intel/customize.py | 6 +- .../customize.py | 11 +++- .../customize.py | 12 ++-- .../customize.py | 6 +- .../app-mlperf-inference-redhat/customize.py | 6 +- script/app-mlperf-inference/customize.py | 29 +++++---- .../customize.py | 4 +- script/benchmark-program/customize.py | 17 ++--- script/build-docker-image/customize.py | 11 ++-- script/build-dockerfile/customize.py | 5 +- script/calibrate-model-for.qaic/customize.py | 9 +-- .../customize.py | 2 +- script/compile-model-for.qaic/customize.py | 6 +- script/compile-program/customize.py | 5 +- script/create-custom-cache-entry/customize.py | 4 +- script/create-patch/customize.py | 12 ++-- script/detect-cpu/customize.py | 10 +-- script/detect-sudo/customize.py | 43 +++++++------ script/download-file/customize.py | 15 +++-- script/extract-file/customize.py | 10 +-- .../customize.py | 39 ++++++----- .../customize.py | 31 +++++---- .../generate-mlperf-tiny-report/customize.py | 6 +- .../customize.py | 17 ++--- script/get-android-sdk/customize.py | 12 ++-- script/get-aocc/customize.py | 5 +- script/get-aria2/customize.py | 9 ++- script/get-aws-cli/customize.py | 5 +- script/get-bazel/customize.py | 6 +- script/get-cl/customize.py | 6 +- script/get-cmake/customize.py | 6 +- script/get-conda/customize.py | 10 ++- script/get-cuda/customize.py | 16 +++-- script/get-cudnn/customize.py | 7 +- script/get-dataset-coco/customize.py | 21 +++--- script/get-dataset-coco2014/customize.py | 1 + .../customize.py | 56 ++++++++-------- script/get-dataset-openimages/customize.py | 6 +- .../customize.py | 23 ++++--- script/get-docker/customize.py | 6 +- script/get-gcc/customize.py | 5 +- script/get-generic-python-lib/customize.py | 1 + script/get-generic-sys-util/customize.py | 21 +++--- script/get-gh-actions-runner/customize.py | 4 +- script/get-github-cli/customize.py | 9 ++- script/get-go/customize.py | 5 +- script/get-huggingface-cli/customize.py | 6 +- script/get-ipol-src/customize.py | 10 +-- script/get-java/customize.py | 11 ++-- script/get-javac/customize.py | 11 ++-- script/get-llvm/customize.py | 5 +- .../customize.py | 4 +- script/get-ml-model-gptj/customize.py | 3 +- script/get-ml-model-mobilenet/customize.py | 3 +- .../customize.py | 9 --- script/get-ml-model-rnnt/customize.py | 4 +- .../customize.py | 6 +- .../customize.py | 14 ++-- script/get-onnxruntime-prebuilt/customize.py | 8 ++- script/get-openssl/customize.py | 5 +- script/get-python3/customize.py | 5 +- script/get-qaic-apps-sdk/customize.py | 7 +- script/get-qaic-platform-sdk/customize.py | 6 +- script/get-rclone/customize.py | 11 +++- script/get-rocm/customize.py | 5 +- script/get-sys-utils-cm/customize.py | 21 +++--- script/get-sys-utils-min/customize.py | 18 +++--- script/get-tensorrt/customize.py | 4 +- script/get-terraform/customize.py | 6 +- script/get-tvm-model/customize.py | 4 +- script/install-bazel/customize.py | 4 +- script/install-cmake-prebuilt/customize.py | 14 ++-- .../install-diffusers-from-src/customize.py | 4 -- script/install-gcc-src/customize.py | 4 +- .../customize.py | 6 +- script/install-gflags/customize.py | 10 +-- script/install-llvm-prebuilt/customize.py | 24 ++++--- script/install-openssl/customize.py | 26 +------- script/install-python-src/customize.py | 4 +- script/install-python-venv/customize.py | 4 +- .../install-rapidjson-from-src/customize.py | 4 -- .../install-torchvision-from-src/customize.py | 4 -- .../plug-prebuilt-cudnn-to-cuda/customize.py | 6 +- .../customize.py | 6 +- .../customize.py | 9 ++- script/process-mlperf-accuracy/customize.py | 16 +++-- script/prune-bert-models/customize.py | 14 ++-- script/push-csv-to-spreadsheet/customize.py | 4 -- .../customize.py | 4 -- script/run-all-mlperf-models/customize.py | 6 +- script/run-docker-container/customize.py | 64 ++++++++++--------- script/run-mlperf-automotive-app/customize.py | 21 +++--- script/run-mlperf-inference-app/customize.py | 22 ++++--- .../customize.py | 7 +- .../customize.py | 3 +- script/run-mlperf-power-client/customize.py | 4 -- script/run-mlperf-power-server/customize.py | 9 +-- script/run-terraform/customize.py | 9 +-- script/run-vllm-server/customize.py | 9 ++- script/runtime-system-infos/customize.py | 8 ++- script/set-venv/customize.py | 19 ++++-- script/submit-mlperf-results/customize.py | 30 ++++----- script/tar-my-folder/customize.py | 5 +- .../customize.py | 3 +- .../customize.py | 4 -- 109 files changed, 661 insertions(+), 490 deletions(-) diff --git a/script/app-image-classification-onnx-py/customize.py b/script/app-image-classification-onnx-py/customize.py index c3c31b17a..39ae86bba 100644 --- a/script/app-image-classification-onnx-py/customize.py +++ b/script/app-image-classification-onnx-py/customize.py @@ -19,7 +19,6 @@ def postprocess(i): os_info = i['os_info'] env = i['env'] state = i['state'] - automation = i['automation'] logger = automation.action_object.logger @@ -48,6 +47,7 @@ def postprocess(i): json.dump(data, f, ensure_ascii=False, indent=4) except Exception as e: logger.warning('CM warning: {}'.format(e)) + logger.warning('CM warning: {}'.format(e)) try: import yaml diff --git a/script/app-image-corner-detection/customize.py b/script/app-image-corner-detection/customize.py index 7b37eb663..8efad95b8 100644 --- a/script/app-image-corner-detection/customize.py +++ b/script/app-image-corner-detection/customize.py @@ -38,6 +38,7 @@ def preprocess(i): def postprocess(i): env = i['env'] - print(env['MLC_OUTPUT'] + " generated in " + env['MLC_RUN_DIR']) + logger = i['automation'].logger + logger.info(env['MLC_OUTPUT'] + " generated in " + env['MLC_RUN_DIR']) return {'return': 0} diff --git a/script/app-loadgen-generic-python/customize.py b/script/app-loadgen-generic-python/customize.py index 9a4a6104e..639ed9c5c 100644 --- a/script/app-loadgen-generic-python/customize.py +++ b/script/app-loadgen-generic-python/customize.py @@ -11,6 +11,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + if 'MLC_ML_MODEL_FILE_WITH_PATH' not in env: return { 'return': 1, 'error': 'Please select a variation specifying the model to run'} @@ -87,9 +89,9 @@ def preprocess(i): env['MLC_RUN_OPTS'] = run_opts - print('') - print('Assembled flags: {}'.format(run_opts)) - print('') + logger.info('') + logger.info('Assembled flags: {}'.format(run_opts)) + logger.info('') return {'return': 0} diff --git a/script/app-mlperf-inference-dummy/customize.py b/script/app-mlperf-inference-dummy/customize.py index f200e915b..93d24a7b1 100644 --- a/script/app-mlperf-inference-dummy/customize.py +++ b/script/app-mlperf-inference-dummy/customize.py @@ -11,6 +11,8 @@ def preprocess(i): return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] + logger = i['automation'].logger + if env.get('MLC_MLPERF_SKIP_RUN', '') == "yes": return {'return': 0} @@ -29,8 +31,8 @@ def preprocess(i): return r run_cmd = r['run_cmd'] run_dir = r['run_dir'] - print(run_cmd) - print(run_dir) + logger.info(run_cmd) + logger.info(run_dir) return {'return': 1, 'error': 'Run command needs to be tested!'} diff --git a/script/app-mlperf-inference-intel/customize.py b/script/app-mlperf-inference-intel/customize.py index 932817163..d6276e5ab 100644 --- a/script/app-mlperf-inference-intel/customize.py +++ b/script/app-mlperf-inference-intel/customize.py @@ -11,6 +11,8 @@ def preprocess(i): return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] + logger = i['automation'].logger + if env.get('MLC_MLPERF_SKIP_RUN', '') == "yes": return {'return': 0} @@ -104,7 +106,7 @@ def preprocess(i): os.path.dirname(env['MLC_ML_MODEL_FILE_WITH_PATH']), 'retinanet-int8-model.pth') elif env['MLC_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE'] == "build_harness": - print(f"Harness Root: {harness_root}") + logger.info(f"Harness Root: {harness_root}") if "bert" in env['MLC_MODEL']: i['run_script_input']['script_name'] = "build_bert_harness" env['MLC_MLPERF_INFERENCE_INTEL_HARNESS_PATH'] = os.path.join( @@ -162,7 +164,7 @@ def preprocess(i): env[model_dir_name]) elif env['MLC_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE'] == "run_harness": - print(f"Harness Root: {harness_root}") + logger.info(f"Harness Root: {harness_root}") if env.get('MLC_MLPERF_LOADGEN_MODE', '') == "compliance": audit_path = env['MLC_MLPERF_INFERENCE_AUDIT_PATH'] shutil.copy(audit_path, env['MLC_RUN_DIR']) diff --git a/script/app-mlperf-inference-mlcommons-cpp/customize.py b/script/app-mlperf-inference-mlcommons-cpp/customize.py index fa9db9d76..36b13cd7c 100644 --- a/script/app-mlperf-inference-mlcommons-cpp/customize.py +++ b/script/app-mlperf-inference-mlcommons-cpp/customize.py @@ -11,10 +11,15 @@ def preprocess(i): meta = i['meta'] + logger = i['automation'].logger + if os_info['platform'] == 'windows': - print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') - print('WARNING: this script was not thoroughly tested on Windows and compilation may fail - please help us test and improve it!') - print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') + logger.info( + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') + logger.warning( + 'This script was not thoroughly tested on Windows and compilation may fail - please help us test and improve it!') + logger.info( + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') # # Currently support only LLVM on Windows # print ('# Forcing LLVM on Windows') # r = automation.update_deps({'deps':meta['post_deps'], 'update_deps':{'compile-program': {'adr':{'compiler':{'tags':'llvm'}}}}}) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index d0eef3b51..8d732b413 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -13,6 +13,8 @@ def preprocess(i): state = i['state'] script_path = i['run_script_input']['path'] + logger = i['automation'].logger + if env.get('MLC_MLPERF_SKIP_RUN', '') == "yes": return {'return': 0} @@ -179,7 +181,7 @@ def preprocess(i): mlperf_implementation = env.get('MLC_MLPERF_IMPLEMENTATION', 'reference') cmd, run_dir = get_run_cmd(os_info, env, scenario_extra_options, - mode_extra_options, dataset_options, mlperf_implementation) + mode_extra_options, dataset_options, logger, mlperf_implementation) if env.get('MLC_NETWORK_LOADGEN', '') == "lon": @@ -202,10 +204,10 @@ def preprocess(i): def get_run_cmd(os_info, env, scenario_extra_options, - mode_extra_options, dataset_options, implementation="reference"): + mode_extra_options, dataset_options, logger, implementation="reference"): if implementation == "reference": return get_run_cmd_reference( - os_info, env, scenario_extra_options, mode_extra_options, dataset_options) + os_info, env, scenario_extra_options, mode_extra_options, dataset_options, logger) if implementation == "nvidia": return get_run_cmd_nvidia( os_info, env, scenario_extra_options, mode_extra_options, dataset_options) @@ -213,7 +215,7 @@ def get_run_cmd(os_info, env, scenario_extra_options, def get_run_cmd_reference( - os_info, env, scenario_extra_options, mode_extra_options, dataset_options): + os_info, env, scenario_extra_options, mode_extra_options, dataset_options, logger): device = env['MLC_MLPERF_DEVICE'] if env['MLC_MLPERF_DEVICE'] not in [ "gpu", "rocm"] else "cuda" @@ -533,7 +535,7 @@ def get_run_cmd_reference( if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '': cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}" - print(cmd) + logger.info(fcmd) if env.get('MLC_NETWORK_LOADGEN', '') in ["lon", "sut"]: cmd = cmd + " " + "--network " + env['MLC_NETWORK_LOADGEN'] diff --git a/script/app-mlperf-inference-qualcomm/customize.py b/script/app-mlperf-inference-qualcomm/customize.py index 2ba7db912..095bfe2dd 100644 --- a/script/app-mlperf-inference-qualcomm/customize.py +++ b/script/app-mlperf-inference-qualcomm/customize.py @@ -8,6 +8,8 @@ def preprocess(i): os_info = i['os_info'] + logger = i['automation'].logger + if os_info['platform'] == 'windows': return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] @@ -27,7 +29,7 @@ def preprocess(i): kilt_root = env['MLC_KILT_CHECKOUT_PATH'] - print(f"Harness Root: {kilt_root}") + logger.info(f"Harness Root: {kilt_root}") source_files = [] env['MLC_SOURCE_FOLDER_PATH'] = env['MLC_KILT_CHECKOUT_PATH'] @@ -161,7 +163,7 @@ def preprocess(i): "master", "QAicInfApi.cpp")) - print(f"Compiling the source files: {source_files}") + logger.info(f"Compiling the source files: {source_files}") env['MLC_CXX_SOURCE_FILES'] = ";".join(source_files) env['+ CXXFLAGS'].append("-std=c++17") diff --git a/script/app-mlperf-inference-redhat/customize.py b/script/app-mlperf-inference-redhat/customize.py index 25ff631d1..67b7b04e4 100644 --- a/script/app-mlperf-inference-redhat/customize.py +++ b/script/app-mlperf-inference-redhat/customize.py @@ -12,6 +12,8 @@ def preprocess(i): return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] + logger = i['automation'].logger + if is_true(env.get('MLC_MLPERF_SKIP_RUN', '')): return {'return': 0} @@ -30,8 +32,8 @@ def preprocess(i): return r run_cmd = r['run_cmd'] run_dir = r['run_dir'] - print(run_cmd) - print(run_dir) + logger.info(run_cmd) + logger.info(run_dir) env['MLC_MLPERF_RUN_CMD'] = run_cmd env['MLC_RUN_DIR'] = run_dir env['MLC_RUN_CMD'] = run_cmd diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index e67e35cf1..5eb368fb1 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -63,6 +63,8 @@ def postprocess(i): state = i['state'] mlc = i['automation'].action_object + logger = i['automation'].logger + # if env.get('MLC_MLPERF_USER_CONF', '') == '': # return {'return': 0} @@ -218,7 +220,7 @@ def postprocess(i): pattern["Offline"] = "Samples per second: (.*)\n" pattern["SingleStream"] = "Mean latency \\(ns\\)\\s*:(.*)" pattern["MultiStream"] = "Mean latency \\(ns\\)\\s*:(.*)" - print("\n") + logger.info("\n") with open("mlperf_log_summary.txt", "r") as fp: summary = fp.read() @@ -241,7 +243,7 @@ def postprocess(i): print( f"SUT: {sut_name}, model: {model_full_name}, scenario: {scenario}, {metric} (mean value) updated as {value}") - print(f"New config stored in {sut_config_path}") + logger.info(f"New config stored in {sut_config_path}") with open(sut_config_path, "w") as f: yaml.dump(sut_config, f) @@ -288,8 +290,8 @@ def postprocess(i): ] = y[1].strip() if not is_false(env.get("MLC_MLPERF_PRINT_SUMMARY", "")): - print("\n") - print(mlperf_log_summary) + logger.info("\n") + logger.info(mlperf_log_summary) with open("measurements.json", "w") as fp: json.dump(measurements, fp, indent=2) @@ -503,7 +505,7 @@ def postprocess(i): else: cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{SCRIPT_PATH}{q} -r {q}{RESULT_DIR}{q} -c {q}{COMPLIANCE_DIR}{q} -o {q}{OUTPUT_DIR}{q}""" - print(cmd) + logger.info(cmd) os.system(cmd) if test == "TEST01": @@ -520,13 +522,13 @@ def postprocess(i): ACCURACY_DIR = os.path.join(RESULT_DIR, "accuracy") if not os.path.exists(ACCURACY_DIR): - print("Accuracy run not yet completed") + logger.warning("Accuracy run not yet completed") return { 'return': 1, 'error': 'TEST01 needs accuracy run to be completed first'} cmd = f"""cd {q}{TEST01_DIR}{q} && bash {q}{SCRIPT_PATH}{q} {q}{os.path.join(ACCURACY_DIR, "mlperf_log_accuracy.json")}{q} {q}{os.path.join(COMPLIANCE_DIR, "mlperf_log_accuracy.json")}{q} """ env['CMD'] = cmd - print(cmd) + logger.info(cmd) r = automation.run_native_script( {'run_script_input': run_script_input, 'env': env, 'script_name': 'verify_accuracy'}) if r['return'] > 0: @@ -538,7 +540,8 @@ def postprocess(i): data = file.read().replace('\n', '\t') if 'TEST PASS' not in data: - print("\nDeterministic TEST01 failed... Trying with non-determinism.\n") + logger.warning( + "\nDeterministic TEST01 failed... Trying with non-determinism.\n") # #Normal test failed, trying the check with non-determinism baseline_accuracy_file = os.path.join( @@ -603,8 +606,8 @@ def postprocess(i): sys_utilisation_log['timestamp']) ''' for i in range(len(sys_utilisation_log['timestamp'])): - print(f"{sys_utilisation_log['timestamp'][i]} {power_begin_time}") - print(sys_utilisation_log['timestamp'][i]>=power_begin_time) + logger.info(f"{sys_utilisation_log['timestamp'][i]} {power_begin_time}") + logger.info(sys_utilisation_log['timestamp'][i]>=power_begin_time) ''' # print(f"{sys_utilisation_log['timestamp'][0]} {power_begin_time}") # print(sys_utilisation_log['timestamp'][0]>=power_begin_time) @@ -616,9 +619,9 @@ def postprocess(i): ) system_utilisation_info_dump["avg_used_memory_gb"] = filtered_log['used_memory_gb'].mean( ) - print("\nSystem utilisation info for the current run:") - print(system_utilisation_info_dump) - print("\n") + logger.info("\nSystem utilisation info for the current run:") + logger.info(system_utilisation_info_dump) + logger.info("\n") if state.get( 'mlperf-inference-implementation') and state['mlperf-inference-implementation'].get('version_info'): diff --git a/script/benchmark-any-mlperf-inference-implementation/customize.py b/script/benchmark-any-mlperf-inference-implementation/customize.py index 00ce47ea5..7f8bb1564 100644 --- a/script/benchmark-any-mlperf-inference-implementation/customize.py +++ b/script/benchmark-any-mlperf-inference-implementation/customize.py @@ -14,6 +14,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) models = env['MODELS'].split(",") @@ -165,7 +167,7 @@ def preprocess(i): with open(os.path.join(script_path, run_file_name + ".sh"), 'w') as f: f.write(run_script_content) - print(run_script_content) + logger.info(run_script_content) run_script_input = i['run_script_input'] r = automation.run_native_script( diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index de0fac475..be97afe39 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -6,7 +6,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - + logger = i['automation'].logger q = '"' if os_info['platform'] == 'windows' else "'" if env.get('MLC_RUN_CMD', '') == '': @@ -104,15 +104,16 @@ def preprocess(i): env['MLC_POST_RUN_CMD'] = post_run_cmd # Print info - print('***************************************************************************') - print('CM script::benchmark-program/run.sh') - print('') - print('Run Directory: {}'.format(env.get('MLC_RUN_DIR', ''))) + logger.info( + '***************************************************************************') + logger.info('CM script::benchmark-program/run.sh') + logger.info('') + logger.info('Run Directory: {}'.format(env.get('MLC_RUN_DIR', ''))) - print('') - print('CMD: {}'.format(env.get('MLC_RUN_CMD', ''))) + logger.info('') + logger.info('CMD: {}'.format(env.get('MLC_RUN_CMD', ''))) - print('') + logger.info('') return {'return': 0} diff --git a/script/build-docker-image/customize.py b/script/build-docker-image/customize.py index acf660261..4b2f35022 100644 --- a/script/build-docker-image/customize.py +++ b/script/build-docker-image/customize.py @@ -8,7 +8,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - + logger = i['automation'].logger dockerfile_path = env.get('MLC_DOCKERFILE_WITH_PATH', '') if dockerfile_path != '' and os.path.exists(dockerfile_path): build_dockerfile = False @@ -83,7 +83,7 @@ def preprocess(i): CMD = ''.join(XCMD) - print(CMD) + logger.info(CMD) env['MLC_DOCKER_BUILD_CMD'] = CMD @@ -102,6 +102,7 @@ def get_image_name(env): def postprocess(i): env = i['env'] + logger = i['automation'].logger # Check if need to push docker image to the Docker Hub if is_true(env.get('MLC_DOCKER_PUSH_IMAGE', '')): @@ -118,12 +119,12 @@ def postprocess(i): with open(dockerfile_path + '.build.bat', 'w') as f: f.write(PCMD + '\n') - print(PCMD) + logger.info(PCMD) - print('') + logger.info('') r = os.system(PCMD) - print('') + logger.info('') if r > 0: return {'return': 1, 'error': 'pushing to Docker Hub failed'} diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 8a01ef753..a3d32b247 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -11,6 +11,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger if env["MLC_DOCKER_OS"] not in ["ubuntu", "rhel", "arch"]: return { @@ -379,7 +380,7 @@ def preprocess(i): env['MLC_DOCKER_RUN_CMD'] = "mlc pull repo && " + \ env['MLC_DOCKER_RUN_CMD'] - print(env['MLC_DOCKER_RUN_CMD']) + logger.info(env['MLC_DOCKER_RUN_CMD']) fake_run = env.get("MLC_DOCKER_FAKE_RUN_OPTION", " --fake_run") + dockerfile_env_input_string fake_run = fake_run + \ @@ -441,7 +442,7 @@ def preprocess(i): s = r['string'] f.write(s + EOL) - print(f"""Dockerfile written at {dockerfile_with_path}""") + logger.info(f"""Dockerfile written at {dockerfile_with_path}""") f.close() diff --git a/script/calibrate-model-for.qaic/customize.py b/script/calibrate-model-for.qaic/customize.py index 706ad1767..42f1c64ca 100644 --- a/script/calibrate-model-for.qaic/customize.py +++ b/script/calibrate-model-for.qaic/customize.py @@ -15,6 +15,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) if is_true(env.get('MLC_CREATE_INPUT_BATCH', '')): @@ -27,7 +29,7 @@ def preprocess(i): return r cmd = r['cmd'] - print("Profiling from " + os.getcwd()) + logger.info("Profiling from " + os.getcwd()) env['MLC_RUN_CMD'] = cmd @@ -214,8 +216,3 @@ def postprocess(i): return {'return': 0} -def get_scale_offset(min_val, max_val): - total_range = max_val - min_val - scale = total_range /256.0 - offset = round(-min_val / scale) - return scale, offset diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py index d1ea84490..948171d40 100644 --- a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py +++ b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py @@ -37,7 +37,7 @@ def preprocess(i): if cache_rm_tags: r = mlc_cache.access({'action': 'rm', 'target': 'cache', 'tags': cache_rm_tags, 'f': True}) - print(r) + logger.info(r) # Check if return code is 0 (success) # currently, the warning code is not being checked as the possibility # arises only for missing cache entry diff --git a/script/compile-model-for.qaic/customize.py b/script/compile-model-for.qaic/customize.py index ca2c48d75..ccbe4143b 100644 --- a/script/compile-model-for.qaic/customize.py +++ b/script/compile-model-for.qaic/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) if env.get('MLC_REGISTER_CACHE', '') == '': @@ -22,7 +24,7 @@ def preprocess(i): return r cmd = r['cmd'] - print("Compiling from " + os.getcwd()) + logger.info("Compiling from " + os.getcwd()) env['MLC_QAIC_MODEL_FINAL_COMPILATION_CMD'] = cmd @@ -39,7 +41,7 @@ def preprocess(i): os.path.join( os.getcwd(), "elfs")) - print(r) + logger.info(r) return {'return': 0} diff --git a/script/compile-program/customize.py b/script/compile-program/customize.py index 8a4a7a669..82f738729 100644 --- a/script/compile-program/customize.py +++ b/script/compile-program/customize.py @@ -6,7 +6,7 @@ def preprocess(i): os_info = i['os_info'] q = '"' if os_info['platform'] == 'windows' else "'" - + logger = i['automation'].logger env = i['env'] CPPFLAGS = env.get('+ CPPFLAGS', []) env['MLC_C_COMPILER_FLAGS'] = " ".join(env.get('+ CFLAGS', []) + CPPFLAGS) @@ -30,7 +30,8 @@ def preprocess(i): # If windows, need to extend it more ... if os_info['platform'] == 'windows' and env.get( 'MLC_COMPILER_FAMILY', '') != 'LLVM': - print("WARNING: compile-program script should be extended to support flags for non-LLVM compilers on Windows") + logger.warning( + "Compile-program script should be extended to support flags for non-LLVM compilers on Windows") return {'return': 0} LDFLAGS = env.get('+ LDFLAGS', []) diff --git a/script/create-custom-cache-entry/customize.py b/script/create-custom-cache-entry/customize.py index a0299f50d..795235bfd 100644 --- a/script/create-custom-cache-entry/customize.py +++ b/script/create-custom-cache-entry/customize.py @@ -7,10 +7,10 @@ def preprocess(i): # CM script internal variables env = i['env'] - + logger = i['automation'].logger extra_cache_tags = [] if env.get('MLC_EXTRA_CACHE_TAGS', '').strip() == '': - print('') + logger.info('') extra_cache_tags_str = input( 'Enter extra tags for the custom CACHE entry separated by comma: ') diff --git a/script/create-patch/customize.py b/script/create-patch/customize.py index a14969634..5b6e83180 100644 --- a/script/create-patch/customize.py +++ b/script/create-patch/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) new_dir = env.get('MLC_CREATE_PATCH_NEW', '') @@ -40,11 +42,11 @@ def preprocess(i): x_exclude, old_dir, new_dir) if not quiet: - print('') - print('Running command:') - print('') - print(cmd) - print('') + logger.info('') + logger.info('Running command:') + logger.info('') + logger.info(cmd) + logger.info('') os.system(cmd) diff --git a/script/detect-cpu/customize.py b/script/detect-cpu/customize.py index bac6e3dc1..b6d1d73fc 100644 --- a/script/detect-cpu/customize.py +++ b/script/detect-cpu/customize.py @@ -35,7 +35,7 @@ def postprocess(i): f = 'tmp-systeminfo.csv' if not os.path.isfile(f): - print('WARNING: {} file was not generated!'.format(f)) + logger.warning('{} file was not generated!'.format(f)) else: keys = {} j = 0 @@ -57,14 +57,14 @@ def postprocess(i): except Exception as e: logger.warning( - 'WARNING: problem processing file {} ({})!'.format( + 'Problem processing file {} ({})!'.format( f, format(e))) pass try: f = 'tmp-wmic-cpu.csv' if not os.path.isfile(f): - logger.warning('WARNING: {} file was not generated!'.format(f)) + logger.warning('{} file was not generated!'.format(f)) else: keys = {} @@ -88,7 +88,7 @@ def postprocess(i): except Exception as e: logger.warning( - 'WARNING: problem processing file {} ({})!'.format( + 'Problem processing file {} ({})!'.format( f, format(e))) pass @@ -103,7 +103,7 @@ def postprocess(i): ########################################################################## # Linux if not os.path.isfile(lscpu_out): - print('WARNING: lscpu.out file was not generated!') + logger.warning('lscpu.out file was not generated!') # Currently ignore this error though probably should fail? # But need to check that is supported on all platforms. diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index 47812286f..a749acd2a 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -19,13 +19,16 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) if os.geteuid() == 0: env['MLC_SUDO'] = '' # root user does not need sudo env['MLC_SUDO_USER'] = "yes" else: - if can_execute_sudo_without_password() or prompt_sudo() == 0: + if can_execute_sudo_without_password( + logger) or prompt_sudo(logger) == 0: env['MLC_SUDO_USER'] = "yes" env['MLC_SUDO'] = 'sudo' @@ -36,7 +39,7 @@ def preprocess(i): return {'return': 0} -def can_execute_sudo_without_password(): +def can_execute_sudo_without_password(logger): try: # Run a harmless command using sudo result = subprocess.run( @@ -53,7 +56,7 @@ def can_execute_sudo_without_password(): else: return False except Exception as e: - print(f"An error occurred: {e}") + logger.error(f"An error occurred: {e}") return False @@ -62,15 +65,17 @@ def reset_terminal(): subprocess.run(['stty', 'sane']) -def prompt_retry(timeout=10, default_retry=False): +def prompt_retry(logger, timeout=10, default_retry=False): """Prompt the user with a yes/no question to retry the command, with a 10-second timeout.""" # Check if we're in an interactive terminal if not sys.stdin.isatty(): if default_retry: - print(f"Non-interactive environment detected. Automatically retrying.") + logger.info( + f"Non-interactive environment detected. Automatically retrying.") else: - print(f"Non-interactive environment detected. Skipping retry.") + logger.warning( + f"Non-interactive environment detected. Skipping retry.") return default_retry # Automatically use the default in non-interactive terminals print( @@ -85,14 +90,14 @@ def prompt_retry(timeout=10, default_retry=False): answer = sys.stdin.readline().strip().lower() if answer in ['y', 'n']: return answer == 'y' # Return True if 'y', False if 'n' - print("\nInvalid input. Please enter 'y' or 'n'.") - return prompt_retry(timeout) # Re-prompt on invalid input + logger.error("\nInvalid input. Please enter 'y' or 'n'.") + return prompt_retry(logger, timeout) # Re-prompt on invalid input else: - print("\nNo input received in 10 seconds. Exiting.") + logger.info("\nNo input received in 10 seconds. Exiting.") return False # No input within the timeout, so don't retry -def is_user_in_sudo_group(): +def is_user_in_sudo_group(logger): """Check if the current user is in the 'sudo' group.""" try: sudo_group = grp.getgrnam('sudo').gr_mem @@ -101,7 +106,7 @@ def is_user_in_sudo_group(): # 'sudo' group doesn't exist (might be different on some systems) return False except Exception as e: - print(f"Error checking sudo group: {str(e)}") + logger.error(f"Error checking sudo group: {str(e)}") return False @@ -123,13 +128,15 @@ def get_input(): return result[0] # Return user input or default -def prompt_sudo(): - if os.geteuid() != 0 and not is_user_in_sudo_group(): # No sudo required for root user +def prompt_sudo(logger): + if os.geteuid() != 0 and not is_user_in_sudo_group( + logger): # No sudo required for root user # Prompt for the password if not os.isatty(sys.stdin.fileno()): - print("Skipping password prompt - non-interactive terminal detected!") + logger.warning( + "Skipping password prompt - non-interactive terminal detected!") password = None else: # password = getpass.getpass("Enter password (-1 to skip): ") @@ -140,7 +147,7 @@ def prompt_sudo(): # Check if the input is -1 if password == "-1": - print("Skipping sudo command.") + logger.warning("Skipping sudo command.") return -1 # Run the command with sudo, passing the password @@ -162,16 +169,16 @@ def prompt_sudo(): ) return 0 except subprocess.TimeoutExpired: - print("Timedout") + logger.info("Timedout") reset_terminal() # Reset terminal to sane state if not prompt_retry(): # If the user chooses not to retry or times out return -1 except subprocess.CalledProcessError as e: - print(f"Command failed: {e.output.decode('utf-8')}") + logger.error(f"Command failed: {e.output.decode('utf-8')}") reset_terminal() # Reset terminal in case of failure return -1 except Exception as e: - print(f"An error occurred: {str(e)}") + logger.error(f"An error occurred: {str(e)}") reset_terminal() # Always reset terminal after error return -1 diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 40fc3c276..6200cc223 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -33,6 +33,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) tool = env.get('MLC_DOWNLOAD_TOOL', '') @@ -64,8 +66,8 @@ def preprocess(i): env['MLC_DOWNLOAD_FILENAME'] = filepath if not quiet: - print('') - print('Using local file: {}'.format(filepath)) + logger.info('') + logger.info('Using local file: {}'.format(filepath)) else: url = env.get('MLC_DOWNLOAD_URL', '') @@ -73,8 +75,8 @@ def preprocess(i): return { 'return': 1, 'error': 'please specify URL using --url={URL} or --env.MLC_DOWNLOAD_URL={URL}'} - print('') - print('Downloading from {}'.format(url)) + logger.info('') + logger.info('Downloading from {}'.format(url)) if '&' in url and tool != "mlcutil": if os_info['platform'] == 'windows': @@ -179,7 +181,8 @@ def preprocess(i): url = env.get('MLC_DOWNLOAD_URL' + str(i), '') if url == '': break - print(f"Download from {oldurl} failed, trying from {url}") + logger.error( + f"Download from {oldurl} failed, trying from {url}") if r['return'] > 0: return r @@ -198,7 +201,7 @@ def preprocess(i): if url == '': break env['MLC_DOWNLOAD_CMD'] += f" || (({del_cmd} {env['MLC_DOWNLOAD_FILENAME']} || true) && wget -nc {extra_download_options} {url})" - print(env['MLC_DOWNLOAD_CMD']) + logger.info(f"{env['MLC_DOWNLOAD_CMD']}") elif tool == "curl": if env.get('MLC_DOWNLOAD_FILENAME', '') != '': diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index f6f041dbc..c0e0957f3 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -22,6 +22,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) filename = env.get('MLC_EXTRACT_FILEPATH', '') @@ -133,10 +135,10 @@ def preprocess(i): env.get('MLC_EXTRACT_TOOL_EXTRA_OPTIONS', '') + \ ' ' + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') + ' ' + x + filename + x - print('') - print('Current directory: {}'.format(os.getcwd())) - print('Command line: "{}"'.format(env['MLC_EXTRACT_CMD'])) - print('') + logger.info('') + logger.info('Current directory: {}'.format(os.getcwd())) + logger.info('Command line: "{}"'.format(env['MLC_EXTRACT_CMD'])) + logger.info('') final_file = env.get('MLC_EXTRACT_EXTRACTED_FILENAME', '') diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 2803fd187..05c681b20 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -56,7 +56,7 @@ def model_in_valid_models(model, mlperf_version): return (True, model) -def generate_submission(env, state, inp, submission_division): +def generate_submission(env, state, inp, submission_division, logger): # Save current user directory cur_dir = os.getcwd() @@ -84,13 +84,13 @@ def generate_submission(env, state, inp, submission_division): submission_dir = env.get('MLC_MLPERF_INFERENCE_SUBMISSION_DIR', '') if env.get('MLC_MLPERF_CLEAN_SUBMISSION_DIR', '') != '': - print('=================================================') + logger.info('=================================================') print( 'Cleaning {} ...'.format( env['MLC_MLPERF_INFERENCE_SUBMISSION_DIR'])) if os.path.exists(submission_dir): shutil.rmtree(submission_dir) - print('=================================================') + logger.info('=================================================') if not os.path.isdir(submission_dir): os.makedirs(submission_dir) @@ -98,8 +98,8 @@ def generate_submission(env, state, inp, submission_division): if is_true(str(env.get('MLC_MLPERF_SUBMISSION_DIR_SHARED', ''))): os.chmod(submission_dir, 0o2775) - print('* MLPerf inference submission dir: {}'.format(submission_dir)) - print('* MLPerf inference results dir: {}'.format(results_dir)) + logger.info('* MLPerf inference submission dir: {}'.format(submission_dir)) + logger.info('* MLPerf inference results dir: {}'.format(results_dir)) results = [ f for f in os.listdir(results_dir) if not os.path.isfile( os.path.join( @@ -137,7 +137,7 @@ def generate_submission(env, state, inp, submission_division): if division not in ['open', 'closed']: return {'return': 1, 'error': '"division" must be "open" or "closed"'} - print('* MLPerf inference division: {}'.format(division)) + logger.info('* MLPerf inference division: {}'.format(division)) path_submission_root = submission_dir path_submission_division = os.path.join(path_submission_root, division) @@ -152,7 +152,7 @@ def generate_submission(env, state, inp, submission_division): submitter = system_meta_default['submitter'] env['MLC_MLPERF_SUBMITTER'] = submitter - print('* MLPerf inference submitter: {}'.format(submitter)) + logger.info('* MLPerf inference submitter: {}'.format(submitter)) if env.get('MLC_MLPERF_SUT_SW_NOTES_EXTRA', '') != '': sw_notes = f"""{system_meta_tmp.get('sw_notes','')} {env['MLC_MLPERF_SUT_SW_NOTES_EXTRA']}""" @@ -281,7 +281,8 @@ def generate_submission(env, state, inp, submission_division): else: new_res = res.replace(" ", "_") - print(f"The SUT folder name for submission generation is: {new_res}") + logger.info( + f"The SUT folder name for submission generation is: {new_res}") platform_prefix = inp.get('platform_prefix', '') if platform_prefix: @@ -338,7 +339,7 @@ def generate_submission(env, state, inp, submission_division): with open(os.path.join(submission_code_path, "README.md"), mode='w') as f: f.write("TBD") # create an empty README - print('* MLPerf inference model: {}'.format(model)) + logger.info('* MLPerf inference model: {}'.format(model)) for scenario in scenarios: # the system_info.txt is copied from the mode directory if # found, else it would be looked under scenario directory @@ -468,7 +469,8 @@ def generate_submission(env, state, inp, submission_division): saved_system_meta_file_path = os.path.join( result_mode_path, "system_meta.json") else: - print("WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. CM is automatically creating one using the system defaults. Please modify them as required.") + logger.error( + "WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. CM is automatically creating one using the system defaults. Please modify them as required.") if os.path.exists(saved_system_meta_file_path): with open(saved_system_meta_file_path, "r") as f: saved_system_meta = json.load(f) @@ -485,7 +487,7 @@ def generate_submission(env, state, inp, submission_division): # system_meta.json is not detected, default one will be # written system_meta = {**system_meta_default, **system_meta} - print(system_meta) + logger.info(system_meta) # check if framework version is there in system_meta, # if not try to fill it from sut_info if system_meta['framework'] == "": @@ -636,7 +638,7 @@ def generate_submission(env, state, inp, submission_division): submission_results_path, "images")) for f in files: - print(' * ' + f) + logger.info(' * ' + f) p_target = os.path.join(submission_results_path, f) shutil.copy( os.path.join( @@ -720,7 +722,7 @@ def generate_submission(env, state, inp, submission_division): result_table, headers = mlperf_utils.get_result_table(results) - print(tabulate(result_table, headers=headers, tablefmt="pretty")) + logger.info(tabulate(result_table, headers=headers, tablefmt="pretty")) sut_readme_file = os.path.join(measurement_path, "README.md") with open(sut_readme_file, mode='w') as f: @@ -733,6 +735,7 @@ def postprocess(i): env = i['env'] state = i['state'] inp = i['input'] + logger = i['automation'].logger submission_divisions = [] @@ -745,12 +748,18 @@ def postprocess(i): # if submission division is not assigned, default value would be taken in # submission_generation function if env.get('MLC_MLPERF_SUBMISSION_DIVISION', '') == '': - r = generate_submission(env, state, inp, submission_division="") + r = generate_submission( + env, + state, + inp, + submission_division="", + logger=logger) if r['return'] > 0: return r else: for submission_division in submission_divisions: - r = generate_submission(env, state, inp, submission_division) + r = generate_submission( + env, state, inp, submission_division, logger=logger) if r['return'] > 0: return r diff --git a/script/generate-mlperf-inference-user-conf/customize.py b/script/generate-mlperf-inference-user-conf/customize.py index bda488003..b08f0b0fb 100644 --- a/script/generate-mlperf-inference-user-conf/customize.py +++ b/script/generate-mlperf-inference-user-conf/customize.py @@ -13,6 +13,7 @@ def preprocess(i): env = i['env'] state = i['state'] script_path = i['run_script_input']['path'] + logger = i['automation'].logger rerun = True if env.get("MLC_RERUN", "") != '' else False @@ -31,7 +32,7 @@ def preprocess(i): env['MLC_MLPERF_LOADGEN_SCENARIO'] = "Offline" if 'MLC_MLPERF_LOADGEN_MODE' not in env: - print("\nNo mode given. Using accuracy as default\n") + logger.info("\nNo mode given. Using accuracy as default\n") env['MLC_MLPERF_LOADGEN_MODE'] = "accuracy" if env.get('OUTPUT_BASE_DIR', '') == '': @@ -140,15 +141,18 @@ def preprocess(i): # if env.get("MLC_MLPERF_FIND_PERFORMANCE_MODE", '') == "yes": if metric == "target_qps": if is_true(env.get("MLC_MLPERF_FIND_PERFORMANCE_MODE", '')): - print("In find performance mode: using 1 as target_qps") + logger.info( + "In find performance mode: using 1 as target_qps") else: - print("No target_qps specified. Using 1 as target_qps") + logger.info( + "No target_qps specified. Using 1 as target_qps") conf[metric] = 1 if metric == "target_latency": if is_true(env.get("MLC_MLPERF_FIND_PERFORMANCE_MODE", '')): - print("In find performance mode: using 0.5ms as target_latency") + logger.info( + "In find performance mode: using 0.5ms as target_latency") else: - print("No target_latency specified. Using default") + logger.info("No target_latency specified. Using default") if is_false(env.get('MLC_MLPERF_USE_MAX_DURATION', 'yes')) or is_true(env.get( 'MLC_MLPERF_MODEL_EQUAL_ISSUE_MODE', 'no')): # Total number of queries needed is a multiple of dataset @@ -288,7 +292,12 @@ def preprocess(i): else: env['MLC_MLPERF_ACCURACY_RESULTS_DIR'] = '' - run_exists = run_files_exist(log_mode, OUTPUT_DIR, required_files, env) + run_exists = run_files_exist( + log_mode, + OUTPUT_DIR, + required_files, + env, + logger) if 'MLC_MLPERF_POWER' in env and env.get( 'MLC_MLPERF_SHORT_RANGING_RUN', '') != 'no' and env['MLC_MLPERF_RUN_STYLE'] == "valid" and mode == "performance": @@ -405,14 +414,14 @@ def preprocess(i): if not run_exists or rerun: - print("Output Dir: '" + OUTPUT_DIR + "'") - print(user_conf) + logger.info("Output Dir: '" + OUTPUT_DIR + "'") + logger.info(user_conf) if env.get('MLC_MLPERF_POWER', '') == "yes" and os.path.exists( env.get('MLC_MLPERF_POWER_LOG_DIR', '')): shutil.rmtree(env['MLC_MLPERF_POWER_LOG_DIR']) else: if not env.get('MLC_MLPERF_COMPLIANCE_RUN_POSTPONED', False): - print("Run files exist, skipping run...\n") + logger.warning("Run files exist, skipping run...\n") env['MLC_MLPERF_SKIP_RUN'] = "yes" if not run_exists or rerun or not measure_files_exist(OUTPUT_DIR, @@ -442,7 +451,7 @@ def preprocess(i): return {'return': 0} -def run_files_exist(mode, OUTPUT_DIR, run_files, env): +def run_files_exist(mode, OUTPUT_DIR, run_files, env, logger): import submission_checker as checker from log_parser import MLPerfLog @@ -516,7 +525,7 @@ def run_files_exist(mode, OUTPUT_DIR, run_files, env): else: cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} {SCRIPT_PATH} -r {RESULT_DIR} -c {COMPLIANCE_DIR} -o {OUTPUT_DIR}" - print(cmd) + logger.info(cmd) os.system(cmd) is_valid = checker.check_compliance_perf_dir(COMPLIANCE_DIR) diff --git a/script/generate-mlperf-tiny-report/customize.py b/script/generate-mlperf-tiny-report/customize.py index 32bc3701d..009d33ac3 100644 --- a/script/generate-mlperf-tiny-report/customize.py +++ b/script/generate-mlperf-tiny-report/customize.py @@ -12,6 +12,8 @@ def preprocess(i): cur_dir = os.getcwd() + logger = i['automation'].logger + # Query cache for results dirs env_repo_tags = env.get('MLC_IMPORT_TINYMLPERF_REPO_TAGS', '').strip() xtags = '' if env_repo_tags == '' else ',version-' + env_repo_tags @@ -49,8 +51,8 @@ def preprocess(i): env['MLC_TINYMLPERF_CURRENT_DIR'] = cur_dir env['MLC_TINYMLPERF_REPO_VERSION'] = version - print('') - print('Repo path: {}'.format(path)) + logger.info('') + logger.info('Repo path: {}'.format(path)) r = automation.run_native_script({'run_script_input': run_script_input, 'env': env, diff --git a/script/generate-mlperf-tiny-submission/customize.py b/script/generate-mlperf-tiny-submission/customize.py index 32a97ef28..ed764a1b2 100644 --- a/script/generate-mlperf-tiny-submission/customize.py +++ b/script/generate-mlperf-tiny-submission/customize.py @@ -17,6 +17,7 @@ def generate_submission(i): env = i['env'] state = i['state'] inp = i['input'] + logger = i['automation'].logger results_dir = env['MLC_MLPERF_RESULTS_DIR'] if 'MLC_MLPERF_SUBMISSION_DIR' not in env: @@ -25,8 +26,8 @@ def generate_submission(i): if not os.path.isdir(submission_dir): os.makedirs(submission_dir) - print('* MLPerf tiny submission dir: {}'.format(submission_dir)) - print('* MLPerf tiny results dir: {}'.format(results_dir)) + logger.info('* MLPerf tiny submission dir: {}'.format(submission_dir)) + logger.info('* MLPerf tiny results dir: {}'.format(results_dir)) results = [ f for f in os.listdir(results_dir) if not os.path.isfile( os.path.join( @@ -40,7 +41,7 @@ def generate_submission(i): system_meta = state['MLC_SUT_META'] division = system_meta['division'] - print('* MLPerf tiny division: {}'.format(division)) + logger.info('* MLPerf tiny division: {}'.format(division)) path_submission_root = submission_dir path_submission_division = os.path.join(path_submission_root, division) @@ -51,7 +52,7 @@ def generate_submission(i): submitter = system_meta['submitter'] env['MLC_MLPERF_SUBMITTER'] = submitter - print('* MLPerf tiny submitter: {}'.format(submitter)) + logger.info('* MLPerf tiny submitter: {}'.format(submitter)) path_submission = os.path.join(path_submission_division, submitter) if not os.path.isdir(path_submission): @@ -67,8 +68,8 @@ def generate_submission(i): target = parts[1] framework = backend - print('* Target: {}'.format(target)) - print('* Framework: {}'.format(framework)) + logger.info('* Target: {}'.format(target)) + logger.info('* Framework: {}'.format(framework)) result_path = os.path.join(results_dir, res) platform_prefix = inp.get('platform_prefix', '') if platform_prefix: @@ -109,7 +110,7 @@ def generate_submission(i): with open(os.path.join(submission_code_path, "README.md"), mode='w'): pass # create an empty README - print('* MLPerf inference model: {}'.format(model)) + logger.info('* MLPerf inference model: {}'.format(model)) for scenario in scenarios: result_scenario_path = os.path.join( result_model_path, scenario) @@ -181,7 +182,7 @@ def generate_submission(i): files.append("accuracy.txt") for f in files: - print(' * ' + f) + logger.info(' * ' + f) p_target = os.path.join(submission_results_path, f) shutil.copy( os.path.join( diff --git a/script/get-android-sdk/customize.py b/script/get-android-sdk/customize.py index 505838b62..eedcb31e6 100644 --- a/script/get-android-sdk/customize.py +++ b/script/get-android-sdk/customize.py @@ -11,6 +11,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] run_script_input = i['run_script_input'] @@ -42,7 +44,7 @@ def preprocess(i): sdk_manager_file = 'sdkmanager' + ext - print('') + logger.info('') found = False @@ -76,8 +78,8 @@ def preprocess(i): env['MLC_ANDROID_CMDLINE_TOOLS_URL'] = package_url - print('') - print('Downloading from {} ...'.format(package_url)) + logger.info('') + logger.info('Downloading from {} ...'.format(package_url)) cm = automation.action_object @@ -89,7 +91,7 @@ def preprocess(i): filename = r['filename'] - print('Unzipping file {}'.format(filename)) + logger.info('Unzipping file {}'.format(filename)) r = cm.access({'action': 'unzip_file', 'automation': 'utils,dc2743f8450541e3', @@ -123,7 +125,7 @@ def preprocess(i): paths.append(sdk_manager_dir) # Prepare SDK - print('Preparing Android SDK manager ...') + logger.info('Preparing Android SDK manager ...') r = automation.run_native_script( {'run_script_input': run_script_input, 'env': env, 'script_name': 'prepare-sdk-manager'}) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 513539438..d5890e103 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -71,6 +71,8 @@ def preprocess(i): def detect_version(i): + logger = i['automation'].logger + r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)', 'group_number': 1, 'env_key': 'MLC_AOCC_VERSION', @@ -79,7 +81,8 @@ def detect_version(i): return r version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + f"{i['recursion_spaces']} Detected version: {version}") return {'return': 0, 'version': version} diff --git a/script/get-aria2/customize.py b/script/get-aria2/customize.py index bb9529ec8..2f7459cde 100644 --- a/script/get-aria2/customize.py +++ b/script/get-aria2/customize.py @@ -8,6 +8,7 @@ def preprocess(i): # Pre-set by CM os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger recursion_spaces = i['recursion_spaces'] automation = i['automation'] run_script_input = i['run_script_input'] @@ -64,7 +65,7 @@ def preprocess(i): version, archive_with_ext) env['MLC_ARIA2_DOWNLOAD_URL'] = url - print('URL to download ARIA2: {}'.format(url)) + logger.info('URL to download ARIA2: {}'.format(url)) r = automation.run_native_script( {'run_script_input': run_script_input, 'env': env, 'script_name': 'install'}) @@ -102,7 +103,7 @@ def preprocess(i): def detect_version(i): env = i['env'] - + logger = i['automation'].logger r = i['automation'].parse_version({'match_text': r'aria2 version\s*([\d.]+)', 'group_number': 1, 'env_key': 'MLC_ARIA2_VERSION', @@ -111,7 +112,9 @@ def detect_version(i): return r version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-aws-cli/customize.py b/script/get-aws-cli/customize.py index 1da3b4d97..24da2b467 100644 --- a/script/get-aws-cli/customize.py +++ b/script/get-aws-cli/customize.py @@ -40,8 +40,11 @@ def detect_version(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-bazel/customize.py b/script/get-bazel/customize.py index 32a629ea3..c60553c17 100644 --- a/script/get-bazel/customize.py +++ b/script/get-bazel/customize.py @@ -41,7 +41,11 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-cl/customize.py b/script/get-cl/customize.py index cd6123150..2c80d4e68 100644 --- a/script/get-cl/customize.py +++ b/script/get-cl/customize.py @@ -123,7 +123,11 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-cmake/customize.py b/script/get-cmake/customize.py index f276ab1bf..ef2047723 100644 --- a/script/get-cmake/customize.py +++ b/script/get-cmake/customize.py @@ -40,7 +40,11 @@ def detect_version(i): return r version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-conda/customize.py b/script/get-conda/customize.py index beb6aa1fb..81829d9d9 100644 --- a/script/get-conda/customize.py +++ b/script/get-conda/customize.py @@ -13,6 +13,8 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] + logger = i['automation'].logger + conda_prefix_name = env.get('MLC_CONDA_PREFIX_NAME', '') r = None file_name = 'conda.exe' if os_info['platform'] == 'windows' else 'conda' @@ -49,7 +51,7 @@ def preprocess(i): if is_true(env.get('MLC_TMP_FAIL_IF_NOT_FOUND', '')): return r - print(recursion_spaces + ' # {}'.format(r['error'])) + logger.error(recursion_spaces + ' # {}'.format(r['error'])) # Attempt to run installer r = automation.run_native_script( @@ -84,7 +86,7 @@ def detect_version(i): def postprocess(i): env = i['env'] - + logger = i['automation'].logger r = detect_version(i) if r['return'] > 0: return r @@ -107,6 +109,8 @@ def postprocess(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-cuda/customize.py b/script/get-cuda/customize.py index 77ac713e9..005de5a2c 100644 --- a/script/get-cuda/customize.py +++ b/script/get-cuda/customize.py @@ -102,7 +102,11 @@ def detect_version_nvcc(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} @@ -110,7 +114,8 @@ def detect_version_nvcc(i): def detect_version_cuda_lib(i): env = i['env'] - print(env) + logger = i['automation'].logger + logger.info(env) cuda_rt_file_path = env['MLC_CUDA_RT_WITH_PATH'] cuda_lib_path = os.path.dirname(cuda_rt_file_path) cuda_path = os.path.abspath(os.path.join(cuda_lib_path, os.pardir)) @@ -128,7 +133,9 @@ def detect_version_cuda_lib(i): env['MLC_CUDA_VERSION'] = cuda_version version = cuda_version - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} @@ -138,6 +145,7 @@ def postprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger r = detect_version(i) if r['return'] > 0: @@ -164,7 +172,7 @@ def postprocess(i): parent_path = os.path.dirname(parent_path) while os.path.isdir(parent_path): if os.path.exists(os.path.join(parent_path, "include")): - print("Path is " + parent_path) + logger.info("Path is " + parent_path) found_path = parent_path cuda_path = found_path env['MLC_CUDA_INSTALLED_PATH'] = cuda_path diff --git a/script/get-cudnn/customize.py b/script/get-cudnn/customize.py index 1e47255e0..b44fbc49d 100644 --- a/script/get-cudnn/customize.py +++ b/script/get-cudnn/customize.py @@ -12,6 +12,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + env['MLC_TMP_RUN_COPY_SCRIPT'] = "no" # If TAR file is not explicitly specified, search @@ -106,7 +108,7 @@ def preprocess(i): return { 'return': 1, 'error': 'Please envoke mlcr get,cudnn --tar_file={full path to the cuDNN tar file}'} - print('Untaring file - can take some time ...') + logger.info('Untaring file - can take some time ...') my_tar = tarfile.open(os.path.expanduser(env['MLC_CUDNN_TAR_FILE_PATH'])) folder_name = my_tar.getnames()[0] @@ -133,7 +135,8 @@ def preprocess(i): print( "Copying cudnn include files to {}(CUDA_INCLUDE_PATH)".format(cuda_inc_path)) shutil.copytree(inc_path, cuda_inc_path, dirs_exist_ok=True) - print("Copying cudnn lib files to {}CUDA_LIB_PATH".format(cuda_lib_path)) + logger.info( + "Copying cudnn lib files to {}CUDA_LIB_PATH".format(cuda_lib_path)) shutil.copytree(lib_path, cuda_lib_path, dirs_exist_ok=True) except BaseException: # Need to copy to system path via run.sh diff --git a/script/get-dataset-coco/customize.py b/script/get-dataset-coco/customize.py index c4d445ac6..e8798fbaf 100644 --- a/script/get-dataset-coco/customize.py +++ b/script/get-dataset-coco/customize.py @@ -11,6 +11,7 @@ def preprocess(i): automation = i['automation'] env = i['env'] meta = i['meta'] + logger = i['automation'].logger quiet = is_true(env.get('MLC_QUIET', False)) # Check if path is there to detect existing data set @@ -38,8 +39,8 @@ def preprocess(i): return { 'return': 1, 'error': 'COCO dataset is not detected in "{}"'.format(path)} - print('') - print('Detected COCO dataset {} {}'.format(tp, ver)) + logger.info('') + logger.info('Detected COCO dataset {} {}'.format(tp, ver)) env['MLC_DATASET_COCO_DETECTED'] = 'yes' env['MLC_DATASET_COCO_PATH'] = path @@ -142,9 +143,9 @@ def preprocess(i): env['MLC_DATASET_COCO_MD5SUM_ANN'] = md5sum_ann if not detected: - print('') - print('URL for data: {}'.format(url_data_full)) - print('URL for annotations: {}'.format(url_ann_full)) + logger.info('') + logger.info('URL for data: {}'.format(url_data_full)) + logger.info('URL for annotations: {}'.format(url_ann_full)) # Add version and type to tags extra_cache_tags = [] @@ -165,6 +166,8 @@ def postprocess(i): path_to = env.get('MLC_TO', '') + logger = i['automation'].logger + # Check if detected or downloaded if env.get('MLC_DATASET_COCO_DETECTED', '').lower() == 'yes' or path_to != '': @@ -181,9 +184,9 @@ def postprocess(i): path_data = env['MLC_DATASET_COCO_DATA_PATH'] path_ann = env['MLC_DATASET_COCO_ANNOTATIONS_PATH'] - print('') - print(path_all) - print('') + logger.info('') + logger.info(path_all) + logger.info('') path_data_full = os.path.join(path_data, tp_ver) path_ann_full = os.path.join(path_ann, 'annotations') @@ -204,7 +207,7 @@ def postprocess(i): command2 = ' ln -s ' + path_ann_full + ' annotations' for command in [command1, command2]: - print(command) + logger.info(command) os.system(command) env['MLC_DATASET_COCO_PATH'] = path_all diff --git a/script/get-dataset-coco2014/customize.py b/script/get-dataset-coco2014/customize.py index d48db2137..9b7f2a467 100644 --- a/script/get-dataset-coco2014/customize.py +++ b/script/get-dataset-coco2014/customize.py @@ -22,6 +22,7 @@ def preprocess(i): def postprocess(i): + logger = i["automation"].logger env = i['env'] if is_true(env.get('MLC_GENERATE_SAMPLE_ID', '')): env['MLC_COCO2014_SAMPLE_ID_PATH'] = os.path.join( diff --git a/script/get-dataset-cognata-mlcommons/customize.py b/script/get-dataset-cognata-mlcommons/customize.py index 937d42de0..1a070e0ab 100644 --- a/script/get-dataset-cognata-mlcommons/customize.py +++ b/script/get-dataset-cognata-mlcommons/customize.py @@ -64,6 +64,8 @@ def postprocess(i): automation = i['automation'] mlc = automation.action_object + logger = automation.logger + cur_dir = os.getcwd() quiet = is_true(env.get('MLC_QUIET', False)) @@ -101,8 +103,8 @@ def postprocess(i): cfg['real_path'] = dataset_path - print('') - print('Used dataset path: {}'.format(dataset_path)) + logger.info('') + logger.info('Used dataset path: {}'.format(dataset_path)) env['MLC_DATASET_MLCOMMONS_COGNATA_PATH'] = dataset_path @@ -121,8 +123,9 @@ def postprocess(i): if cfg.get('processed', False): if not utils.check_if_true_yes_on( env, 'MLC_DATASET_MLCOMMONS_COGNATA_UPDATE'): - print('') - print('Already processed: use --update to update this dataset') + logger.info('') + logger.info( + 'Already processed: use --update to update this dataset') return {'return': 0} @@ -152,7 +155,7 @@ def postprocess(i): if x != '': first_url = x else: - print('') + logger.info('') first_url = input( 'Please register at https://mlcommons.org/datasets/cognata and enter private URL: ') @@ -232,12 +235,13 @@ def postprocess(i): return {'return': 0, 'error': 'no sets found'} ########################################################################## - print('') - print('Available or selected serial numbers (use --serial_numbers=a,b,c to download specific subsets):') - print('') + logger.info('') + logger.info( + 'Available or selected serial numbers (use --serial_numbers=a,b,c to download specific subsets):') + logger.info('') for d in data: s = d[serial_key] - print(s) + logger.info(s) for d in data: url = d[url_key] @@ -253,8 +257,8 @@ def postprocess(i): if not os.path.isfile(dataset_path2): - print('') - print('Downloading {} ...'.format(url_export)) + logger.info('') + logger.info('Downloading {} ...'.format(url_export)) r = mlc.access({'action': 'run', 'automation': 'script', @@ -267,8 +271,8 @@ def postprocess(i): return r ########################################################################## - print('') - print('Processing subsets ...') + logger.info('') + logger.info('Processing subsets ...') group_names = [] for s in env.get('MLC_DATASET_MLCOMMONS_COGNATA_GROUP_NAMES', @@ -289,8 +293,8 @@ def postprocess(i): dataset_path2 = os.path.join(dataset_path1, serial_file) dataset_path3 = os.path.join(dataset_path1, d[serial_key]) - print('') - print('Processing {} ...'.format(serial_file)) + logger.info('') + logger.info('Processing {} ...'.format(serial_file)) dataset_key = 'File_Data' url_key = 'File_Link' @@ -332,12 +336,12 @@ def postprocess(i): url = d[url_key] - print('') - print('Downloading {} ...'.format(file_name)) + logger.info('') + logger.info('Downloading {} ...'.format(file_name)) if os.path.isfile(file_name_with_path_done): - print('') - print(' Already processed - skipping ...') + logger.info('') + logger.warning(' Already processed - skipping ...') continue if os.name == 'nt': @@ -349,16 +353,16 @@ def postprocess(i): ' --async-dns=false -x15 -s15 "{}" --dir "{}" -o "{}"'.format( url, dataset_path3, file_name) - print('') - print(cmd) - print('') + logger.info('') + logger.info(cmd) + logger.info('') os.system(cmd) # Unarchive - print('') - print('Extracting file {} ...'.format(file_name_with_path)) - print('') + logger.info('') + logger.info('Extracting file {} ...'.format(file_name_with_path)) + logger.info('') if file_name.endswith('.zip'): @@ -390,7 +394,7 @@ def postprocess(i): # Remove file os.remove(file_name_with_path) - print('') + logger.info('') # Mark that processed this dataset once correctly cfg['processed'] = True diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 9d4539739..b0a23bec5 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -8,11 +8,11 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - - print("") + logger = i['automation'].logger + logger.info("") print("Using MLCommons Inference source from '" + env['MLC_MLPERF_INFERENCE_SOURCE'] + "'") - print("") + logger.info("") if os_info['platform'] == 'windows': MLPERF_CLASSES = ['Airplane', 'Antelope', 'Apple', 'Backpack', 'Balloon', 'Banana', diff --git a/script/get-dlrm-data-mlperf-inference/customize.py b/script/get-dlrm-data-mlperf-inference/customize.py index 8f0153094..2271c5a36 100644 --- a/script/get-dlrm-data-mlperf-inference/customize.py +++ b/script/get-dlrm-data-mlperf-inference/customize.py @@ -9,11 +9,13 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + dlrm_data_path = env.get( 'MLC_DLRM_DATA_PATH', env.get( 'DLRM_DATA_PATH', '')) if dlrm_data_path == '': - print( + logger.warning( f'Data path is not given as input through --dlrm_data_path. Using the cache directory:{os.getcwd()} as the data path') dlrm_data_path = os.getcwd() elif not os.path.exists(dlrm_data_path): @@ -42,37 +44,40 @@ def preprocess(i): if variation == "nvidia": if not os.path.exists(os.path.join(dlrm_data_path, "model")): - print(f'model directory is missing inside {dlrm_data_path}') + logger.warning( + f'model directory is missing inside {dlrm_data_path}') env['MLC_DLRM_MODEL_DOWNLOAD'] = True if not os.path.exists(os.path.join(dlrm_data_path, "criteo")): - print(f'criteo directory is missing inside {dlrm_data_path}') + logger.warning( + f'criteo directory is missing inside {dlrm_data_path}') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join( dlrm_data_path, "model", "model_weights")): - print( + logger.warning( f'model_weights directory is missing inside {dlrm_data_path}/model') env['MLC_DLRM_MODEL_DOWNLOAD'] = True if not os.path.exists(os.path.join(dlrm_data_path, "criteo", "day23")): - print(f'day23 directory is missing inside {dlrm_data_path}/day23') + logger.warning( + f'day23 directory is missing inside {dlrm_data_path}/day23') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join( dlrm_data_path, "criteo", "day23", "fp32")): - print( + logger.warning( f'fp32 directory is missing inside {dlrm_data_path}/criteo/day23') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join(dlrm_data_path, "criteo", "day23", "fp32", "day_23_sparse_multi_hot.npz")) and not os.path.exists( os.path.join(dlrm_data_path, "criteo", "day23", "fp32", "day_23_sparse_multi_hot_unpacked")): - print( + logger.warning( f'day_23_sparse_multi_hot.npz or day_23_sparse_multi_hot_unpacked is missing inside {dlrm_data_path}/criteo/day23/fp32') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join( dlrm_data_path, "criteo", "day23", "fp32", "day_23_dense.npy")): - print( + logger.warning( f'day_23_dense.npy is missing inside {dlrm_data_path}/criteo/day23/fp32') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join( dlrm_data_path, "criteo", "day23", "fp32", "day_23_labels.npy")): - print( + logger.warning( f'day_23_labels.npy is missing inside {dlrm_data_path}/criteo/day23/fp32') env['MLC_DLRM_DATASET_DOWNLOAD'] = True if not os.path.exists(os.path.join( diff --git a/script/get-docker/customize.py b/script/get-docker/customize.py index 3097e343f..4f5309e97 100644 --- a/script/get-docker/customize.py +++ b/script/get-docker/customize.py @@ -64,6 +64,8 @@ def detect_version(i): if r['return'] > 0: return r + logger = i['automation'].logger + version = r['version'] tool = "docker" @@ -71,7 +73,9 @@ def detect_version(i): if "podman" in r['string'].lower(): tool = "podman" - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version, "tool": tool} diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 01749aba5..25e39a781 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -59,8 +59,11 @@ def detect_version(i): return {'return': 0, 'version': -1} return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index 02c8071c6..4ce85258b 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -9,6 +9,7 @@ def preprocess(i): env = i['env'] meta = i['meta'] automation = i['automation'] + logger = automation.logger run_script_input = i['run_script_input'] pip_version = env.get('MLC_PIP_VERSION', '').strip().split('.') diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index f7c7a37d0..250afdbdc 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,7 @@ def preprocess(i): env = i['env'] state = i['state'] automation = i['automation'] - + logger = automation.logger # Use VERSION_CMD as CHECK_CMD if no CHECK_CMD is set if env.get('MLC_SYS_UTIL_VERSION_CMD', '') != '' and env.get( 'MLC_SYS_UTIL_CHECK_CMD', '') == '': @@ -29,7 +29,7 @@ def preprocess(i): r = automation.run_native_script( {'run_script_input': i['run_script_input'], 'env': env, 'script_name': 'detect'}) if r['return'] != 0: # detection failed, do install via prehook_deps - print("detection failed, going for installation") + logger.warning("detection failed, going for installation") env['MLC_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} else: # detection is successful, no need to install @@ -54,9 +54,9 @@ def preprocess(i): package_name = package.get(pm) if os_info['platform'] == 'windows' and not package_name: - print('') - print('WARNING: for now skipping get-generic-sys-util on Windows ...') - print('') + logger.info('') + logger.warning('For now skipping get-generic-sys-util on Windows ...') + logger.info('') return {'return': 0} @@ -68,9 +68,10 @@ def preprocess(i): 'error': f'No package name specified for {util} in the meta'} if not package_name: - if is_true(env.get('MLC_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', False)): - print( - f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...") + if str(env.get('MLC_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '') + ).lower() in ["1", "true", "yes"]: + logger.warning( + f"No package name specified for {pm} and util name {util}. Ignoring it...") env['MLC_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED'] = 'yes' return {'return': 0} else: @@ -130,7 +131,7 @@ def detect_version(i): version_env_key = f"MLC_{env['MLC_SYS_UTIL_NAME'].upper()}_VERSION" version_check_re = env.get('MLC_SYS_UTIL_VERSION_RE', '') group_number = env.get('MLC_TMP_VERSION_DETECT_GROUP_NUMBER', 1) - + logger = i['automation'].logger # Confirm that the regex pattern and file are present if version_check_re == '' or not os.path.exists("tmp-ver.out"): version = "undetected" @@ -144,7 +145,7 @@ def detect_version(i): return r version = r['version'] - print( + logger.info( i['recursion_spaces'] + ' Detected version: {}'.format(version)) diff --git a/script/get-gh-actions-runner/customize.py b/script/get-gh-actions-runner/customize.py index 795c50da6..09d8f8330 100644 --- a/script/get-gh-actions-runner/customize.py +++ b/script/get-gh-actions-runner/customize.py @@ -13,7 +13,7 @@ def preprocess(i): automation = i['automation'] mlc = automation.action_object - + logger = automation.logger quiet = is_true(env.get('MLC_QUIET', False)) cmd = env.get('MLC_GH_ACTIONS_RUNNER_COMMAND', '') @@ -28,7 +28,7 @@ def preprocess(i): cache_rm_tags = "gh,runner,_install" r = mlc.access({'action': 'rm', 'automation': 'cache', 'tags': cache_rm_tags, 'f': True}) - print(r) + logger.info(r) if r['return'] != 0 and r['return'] != 16: # ignore missing ones return r elif cmd == "start": diff --git a/script/get-github-cli/customize.py b/script/get-github-cli/customize.py index 7993b1e3c..4ce75bd43 100644 --- a/script/get-github-cli/customize.py +++ b/script/get-github-cli/customize.py @@ -10,7 +10,7 @@ def preprocess(i): env = i['env'] recursion_spaces = i['recursion_spaces'] - + logger = i['automation'].logger file_name = 'gh.exe' if os_info['platform'] == 'windows' else 'gh' # Will check env['MLC_TMP_PATH'] if comes from installation script @@ -27,7 +27,7 @@ def preprocess(i): if is_true(env.get('MLC_TMP_FAIL_IF_NOT_FOUND', '')): return r - print(recursion_spaces + ' # {}'.format(r['error'])) + logger.error(recursion_spaces + ' # {}'.format(r['error'])) # Attempt to run installer r = { @@ -54,7 +54,10 @@ def postprocess(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-go/customize.py b/script/get-go/customize.py index 95ff8630a..241aa149d 100644 --- a/script/get-go/customize.py +++ b/script/get-go/customize.py @@ -40,8 +40,11 @@ def detect_version(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-huggingface-cli/customize.py b/script/get-huggingface-cli/customize.py index ee0a1ff42..990c17c28 100644 --- a/script/get-huggingface-cli/customize.py +++ b/script/get-huggingface-cli/customize.py @@ -16,7 +16,7 @@ def preprocess(i): def postprocess(i): env = i['env'] - + logger = i['automation'].logger r = i['automation'].parse_version({'match_text': r'huggingface_hub\s*version:\s*([\d.]+)', 'group_number': 1, 'env_key': 'MLC_GITHUBCLI_VERSION', @@ -26,6 +26,8 @@ def postprocess(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-ipol-src/customize.py b/script/get-ipol-src/customize.py index 4de9d4f9c..a010447c4 100644 --- a/script/get-ipol-src/customize.py +++ b/script/get-ipol-src/customize.py @@ -11,7 +11,7 @@ def preprocess(i): script_path = i['artifact'].path automation = i['automation'] - + logger = automation.logger cm = automation.action_object path = os.getcwd() @@ -27,7 +27,7 @@ def preprocess(i): '{{MLC_IPOL_NUMBER}}', number) - print('Downloading from {}'.format(url)) + logger.info('Downloading from {}'.format(url)) r = cm.access({'action': 'download_file', 'automation': 'utils,dc2743f8450541e3', @@ -37,7 +37,7 @@ def preprocess(i): filename = r['filename'] - print('Unzipping file {}'.format(filename)) + logger.info('Unzipping file {}'.format(filename)) r = cm.access({'action': 'unzip_file', 'automation': 'utils,dc2743f8450541e3', @@ -46,7 +46,7 @@ def preprocess(i): return r if os.path.isfile(filename): - print('Removing file {}'.format(filename)) + logger.info('Removing file {}'.format(filename)) os.remove(filename) # Get sub-directory from filename @@ -60,7 +60,7 @@ def preprocess(i): cmd = 'patch -p0 < {}'.format(os.path.join(script_path, 'patch', '20240127.patch')) - print('Patching code: {}'.format(cmd)) + logger.info('Patching code: {}'.format(cmd)) os.system(cmd) return {'return': 0} diff --git a/script/get-java/customize.py b/script/get-java/customize.py index c646f20b8..3b8145d45 100644 --- a/script/get-java/customize.py +++ b/script/get-java/customize.py @@ -10,7 +10,7 @@ def preprocess(i): env = i['env'] automation = i['automation'] - + logger = automation.logger recursion_spaces = i['recursion_spaces'] run_script_input = i['run_script_input'] @@ -68,7 +68,7 @@ def preprocess(i): env['MLC_JAVA_PREBUILT_URL'] = url env['MLC_JAVA_PREBUILT_FILENAME'] = filename - print('') + logger.info('') print( recursion_spaces + ' Downloading and installing prebuilt Java from {} ...'.format( @@ -88,7 +88,7 @@ def preprocess(i): return {'return': 1, 'error': 'can\'t find target file {}'.format(target_file)} - print('') + logger.info('') print( recursion_spaces + ' Registering file {} ...'.format(target_file)) @@ -126,8 +126,11 @@ def detect_version(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-javac/customize.py b/script/get-javac/customize.py index 7376a523e..997d9dfff 100644 --- a/script/get-javac/customize.py +++ b/script/get-javac/customize.py @@ -10,7 +10,7 @@ def preprocess(i): env = i['env'] automation = i['automation'] - + logger = automation.logger recursion_spaces = i['recursion_spaces'] run_script_input = i['run_script_input'] @@ -68,7 +68,7 @@ def preprocess(i): env['MLC_JAVAC_PREBUILT_URL'] = url env['MLC_JAVAC_PREBUILT_FILENAME'] = filename - print('') + logger.info('') print( recursion_spaces + ' Downloading and installing prebuilt Java from {} ...'.format( @@ -88,7 +88,7 @@ def preprocess(i): return {'return': 1, 'error': 'can\'t find target file {}'.format(target_file)} - print('') + logger.info('') print( recursion_spaces + ' Registering file {} ...'.format(target_file)) @@ -126,8 +126,11 @@ def detect_version(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-llvm/customize.py b/script/get-llvm/customize.py index e53a7719f..070f8d436 100644 --- a/script/get-llvm/customize.py +++ b/script/get-llvm/customize.py @@ -56,8 +56,11 @@ def detect_version(i): return r version = r['version'] + logger = i['automation'].logger - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-ml-model-efficientnet-lite/customize.py b/script/get-ml-model-efficientnet-lite/customize.py index b9a547fee..1f3568ddc 100644 --- a/script/get-ml-model-efficientnet-lite/customize.py +++ b/script/get-ml-model-efficientnet-lite/customize.py @@ -10,6 +10,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + cm = automation.action_object path = os.getcwd() @@ -17,7 +19,7 @@ def preprocess(i): url = env['MLC_PACKAGE_URL'] env['MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME'] = url - print('Downloading from {}'.format(url)) + logger.info('Downloading from {}'.format(url)) r = download_file({'url': url}) if r['return'] > 0: diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index 997c133ed..8d0ca4e9c 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -7,6 +7,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger if env.get('MLC_GPTJ_INTEL_MODEL', '') == 'yes': i['run_script_input']['script_name'] = 'run-intel' @@ -17,7 +18,7 @@ def preprocess(i): 'code', 'gptj-99', 'pytorch-cpu') - print(f"Harness Root: {harness_root}") + logger.info(f"Harness Root: {harness_root}") env['MLC_HARNESS_CODE_ROOT'] = harness_root env['MLC_CALIBRATION_CODE_ROOT'] = os.path.join( env['MLC_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration') diff --git a/script/get-ml-model-mobilenet/customize.py b/script/get-ml-model-mobilenet/customize.py index b9a547fee..cbafeb19a 100644 --- a/script/get-ml-model-mobilenet/customize.py +++ b/script/get-ml-model-mobilenet/customize.py @@ -9,6 +9,7 @@ def preprocess(i): env = i['env'] automation = i['automation'] + logger = automation.logger cm = automation.action_object @@ -17,7 +18,7 @@ def preprocess(i): url = env['MLC_PACKAGE_URL'] env['MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME'] = url - print('Downloading from {}'.format(url)) + logger.info('Downloading from {}'.format(url)) r = download_file({'url': url}) if r['return'] > 0: diff --git a/script/get-ml-model-retinanet-nvidia/customize.py b/script/get-ml-model-retinanet-nvidia/customize.py index efcc67e3d..da2c4adeb 100644 --- a/script/get-ml-model-retinanet-nvidia/customize.py +++ b/script/get-ml-model-retinanet-nvidia/customize.py @@ -27,12 +27,3 @@ def preprocess(i): "onnx_generator", "retinanet_anchor_xywh_1x1.npy") return {'return': 0} - - -def postprocess(i): - env = i['env'] - env['MLC_NVIDIA_RETINANET_EFFICIENT_NMS_CONCAT_MODEL_WITH_PATH'] = os.path.join( - os.getcwd(), "test_fpn_efficientnms_concatall.onnx") - if "MLC_NVIDIA_EFFICIENT_NMS" in env: - env['MLC_NVIDIA_RETINANET_EFFICIENT_NMS_CONCAT_MODEL_WITH_PATH'] = env['MLC_NVIDIA_MODEL_PATCHED_PATH'] - return {'return': 0} diff --git a/script/get-ml-model-rnnt/customize.py b/script/get-ml-model-rnnt/customize.py index a63eb152f..c6ec5aefa 100644 --- a/script/get-ml-model-rnnt/customize.py +++ b/script/get-ml-model-rnnt/customize.py @@ -11,13 +11,15 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + cm = automation.action_object path = os.getcwd() url = env['MLC_PACKAGE_URL'] - print('Downloading from {}'.format(url)) + logger.info('Downloading from {}'.format(url)) r = cm.access({'action': 'download_file', 'automation': 'utils,dc2743f8450541e3', diff --git a/script/get-mlperf-inference-sut-description/customize.py b/script/get-mlperf-inference-sut-description/customize.py index 4c35209e9..9a425eced 100644 --- a/script/get-mlperf-inference-sut-description/customize.py +++ b/script/get-mlperf-inference-sut-description/customize.py @@ -10,6 +10,8 @@ def preprocess(i): state = i['state'] os_info = i['os_info'] + logger = i['automation'].logger + submitter = env.get('MLC_MLPERF_SUBMITTER', 'MLCommons') auto_detected_hw_name = False @@ -45,13 +47,13 @@ def preprocess(i): env['MLC_SUT_PATH'] = sut_path if os.path.exists(sut_path) and is_true(env.get('MLC_SUT_DESC_CACHE', '')): - print(f"Reusing SUT description file {sut}") + logger.info(f"Reusing SUT description file {sut}") state['MLC_SUT_META'] = json.load(open(sut_path)) else: if not os.path.exists(os.path.dirname(sut_path)): os.makedirs(os.path.dirname(sut_path)) - print("Generating SUT description file for " + sut) + logger.info("Generating SUT description file for " + sut) hw_path = os.path.join(os.getcwd(), "hardware", hw_name + ".json") if not os.path.exists(os.path.dirname(hw_path)): os.makedirs(os.path.dirname(hw_path)) diff --git a/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py b/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py index ae9548c8b..157c6b9f8 100644 --- a/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py +++ b/script/get-mlperf-tiny-eembc-energy-runner-src/customize.py @@ -25,6 +25,8 @@ def postprocess(i): env = i['env'] state = i['state'] + logger = i['automation'].logger + env['MLC_EEMBC_ENERGY_RUNNER_SRC'] = os.path.join(os.getcwd(), 'src') datasets_src_path = os.path.join(os.getcwd(), 'src', 'datasets') env['MLC_EEMBC_ENERGY_RUNNER_SRC_DATASETS'] = datasets_src_path @@ -34,8 +36,8 @@ def postprocess(i): sessions_path = os.path.join(home_directory, 'eembc', 'runner', 'sessions') - print('') - print('Path to EEMBC runner sessions: {}'.format(sessions_path)) + logger.info('') + logger.info('Path to EEMBC runner sessions: {}'.format(sessions_path)) env['MLC_EEMBC_ENERGY_RUNNER_SESSIONS'] = sessions_path @@ -50,16 +52,16 @@ def postprocess(i): 'ulp-mlperf', 'datasets') - print('') - print('Path to EEMBC runner datasets: {}'.format(datasets_path)) + logger.info('') + logger.info('Path to EEMBC runner datasets: {}'.format(datasets_path)) if not os.path.isdir(datasets_path): os.makedirs(datasets_path) env['MLC_EEMBC_ENERGY_RUNNER_DATASETS'] = datasets_path - print('') - print('Copying datasets to EEMBC user space ...') + logger.info('') + logger.info('Copying datasets to EEMBC user space ...') shutil.copytree(datasets_src_path, datasets_path, dirs_exist_ok=True) diff --git a/script/get-onnxruntime-prebuilt/customize.py b/script/get-onnxruntime-prebuilt/customize.py index b6045ddd0..3934128d1 100644 --- a/script/get-onnxruntime-prebuilt/customize.py +++ b/script/get-onnxruntime-prebuilt/customize.py @@ -6,6 +6,8 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger + machine = env.get('MLC_HOST_OS_MACHINE', '') if machine == '': machine = 'x86_64' @@ -35,9 +37,9 @@ def preprocess(i): URL = 'https://github.com/microsoft/onnxruntime/releases/download/v{}/{}'.format( version, FILENAME) - print('') - print('Downloading from {}'.format(URL)) - print('') + logger.info('') + logger.info('Downloading from {}'.format(URL)) + logger.info('') env['FOLDER'] = FOLDER env['FILENAME'] = FILENAME diff --git a/script/get-openssl/customize.py b/script/get-openssl/customize.py index 6cdb4a657..41b2416e7 100644 --- a/script/get-openssl/customize.py +++ b/script/get-openssl/customize.py @@ -39,7 +39,10 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-python3/customize.py b/script/get-python3/customize.py index 299dc0b59..64ed74c8e 100644 --- a/script/get-python3/customize.py +++ b/script/get-python3/customize.py @@ -65,7 +65,10 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-qaic-apps-sdk/customize.py b/script/get-qaic-apps-sdk/customize.py index 289c2a70b..ae702daf7 100644 --- a/script/get-qaic-apps-sdk/customize.py +++ b/script/get-qaic-apps-sdk/customize.py @@ -43,6 +43,9 @@ def preprocess(i): def detect_version(i): env = i['env'] + + logger = i['automation'].logger + sdk_path = env['MLC_QAIC_APPS_SDK_PATH'] version = None version_xml_path = os.path.join(sdk_path, "versions", "apps.xml") @@ -64,7 +67,9 @@ def detect_version(i): if not version: return {'return': 1, 'error': f'qaic apps sdk version info not found'} - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-qaic-platform-sdk/customize.py b/script/get-qaic-platform-sdk/customize.py index da006b802..597e4a1fb 100644 --- a/script/get-qaic-platform-sdk/customize.py +++ b/script/get-qaic-platform-sdk/customize.py @@ -44,6 +44,8 @@ def preprocess(i): def detect_version(i): env = i['env'] + + logger = i['automation'].logger sdk_path = env['MLC_QAIC_PLATFORM_SDK_PATH'] version = None version_xml_path = os.path.join(sdk_path, "versions", "platform.xml") @@ -65,7 +67,9 @@ def detect_version(i): if not version: return {'return': 1, 'error': f'qaic platform sdk version info not found'} - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py index 0b3a05065..91fa9b685 100644 --- a/script/get-rclone/customize.py +++ b/script/get-rclone/customize.py @@ -91,7 +91,10 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} @@ -101,6 +104,8 @@ def postprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger + gdrive = env.get('MLC_RCLONE_GDRIVE', '') if gdrive == "yes": config = configparser.ConfigParser() @@ -122,8 +127,8 @@ def postprocess(i): with open(default_config_path, 'w') as configfile: default_config.write(configfile) - print({section: dict(default_config[section]) - for section in default_config.sections()}) + logger.info( + f"{section: dict(default_config[section]) for section in default_config.sections()}") r = detect_version(i) diff --git a/script/get-rocm/customize.py b/script/get-rocm/customize.py index 351c338aa..8dac34e7f 100644 --- a/script/get-rocm/customize.py +++ b/script/get-rocm/customize.py @@ -43,7 +43,10 @@ def detect_version(i): version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger = i['automation'].logger + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-sys-utils-cm/customize.py b/script/get-sys-utils-cm/customize.py index dd2672fd8..ed72dac02 100644 --- a/script/get-sys-utils-cm/customize.py +++ b/script/get-sys-utils-cm/customize.py @@ -9,6 +9,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + automation = i['automation'] cm = automation.action_object @@ -25,9 +27,9 @@ def preprocess(i): # "detect,os"! if os_info['platform'] == 'windows': - print('') - print('This script is not used on Windows') - print('') + logger.info('') + logger.warning('This script is not used on Windows') + logger.info('') # If windows, download here otherwise use run.sh @@ -81,10 +83,13 @@ def preprocess(i): # env['+PATH']=[os.path.join(path, 'bin')] # else: - print('') - print('***********************************************************************') - print('This script will attempt to install minimal system dependencies for CM.') - print('Note that you may be asked for your SUDO password ...') - print('***********************************************************************') + logger.info('') + logger.info( + '***********************************************************************') + logger.info( + 'This script will attempt to install minimal system dependencies for CM.') + logger.info('Note that you may be asked for your SUDO password ...') + logger.info( + '***********************************************************************') return {'return': 0} diff --git a/script/get-sys-utils-min/customize.py b/script/get-sys-utils-min/customize.py index ed1a2519b..17f93a1aa 100644 --- a/script/get-sys-utils-min/customize.py +++ b/script/get-sys-utils-min/customize.py @@ -9,6 +9,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + automation = i['automation'] cm = automation.action_object @@ -23,22 +25,22 @@ def preprocess(i): for cd in clean_dirs.split(','): if cd != '': if os.path.isdir(cd): - print('Cleaning directory {}'.format(cd)) + logger.info('Cleaning directory {}'.format(cd)) shutil.rmtree(cd) url = env['MLC_PACKAGE_WIN_URL'] urls = [url] if ';' not in url else url.split(';') - print('') - print('Current directory: {}'.format(os.getcwd())) + logger.info('') + logger.info('Current directory: {}'.format(os.getcwd())) for url in urls: url = url.strip() - print('') - print('Downloading from {}'.format(url)) + logger.info('') + logger.info('Downloading from {}'.format(url)) r = download_file({ 'url': url, 'verify': False}) @@ -47,7 +49,7 @@ def preprocess(i): filename = r['filename'] - print('Unzipping file {}'.format(filename)) + logger.info('Unzipping file {}'.format(filename)) r = unzip_file({ 'filename': filename}) @@ -55,10 +57,10 @@ def preprocess(i): return r if os.path.isfile(filename): - print('Removing file {}'.format(filename)) + logger.info('Removing file {}'.format(filename)) os.remove(filename) - print('') + logger.info('') # Add to path env['+PATH'] = [os.path.join(path, 'bin')] diff --git a/script/get-tensorrt/customize.py b/script/get-tensorrt/customize.py index 616111509..9eeb536d7 100644 --- a/script/get-tensorrt/customize.py +++ b/script/get-tensorrt/customize.py @@ -12,6 +12,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + # Not enforcing dev requirement for now if env.get('MLC_TENSORRT_TAR_FILE_PATH', '') == '' and env.get( 'MLC_TENSORRT_REQUIRE_DEV1', '') != 'yes' and env.get('MLC_HOST_PLATFORM_FLAVOR_', '') != 'aarch64': @@ -95,7 +97,7 @@ def preprocess(i): return {'return': 1, 'error': 'Please envoke mlcr "' + ",".join(tags) + '" --tar_file={full path to the TensorRT tar file}'} - print('Untaring file - can take some time ...') + logger.info('Untaring file - can take some time ...') file_name = "trtexec" my_tar = tarfile.open( diff --git a/script/get-terraform/customize.py b/script/get-terraform/customize.py index 39ae728c2..eeb1c13b7 100644 --- a/script/get-terraform/customize.py +++ b/script/get-terraform/customize.py @@ -39,9 +39,13 @@ def detect_version(i): if r['return'] > 0: return r + logger = i['automation'].logger + version = r['version'] - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/get-tvm-model/customize.py b/script/get-tvm-model/customize.py index f214f570e..8dafcfba6 100644 --- a/script/get-tvm-model/customize.py +++ b/script/get-tvm-model/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) work_dir = env.get('MLC_TUNE_TVM_MODEL_WORKDIR', '') @@ -31,7 +33,7 @@ def preprocess(i): "Error: the found workdir does not contain database_tuning_record.json") if env.get('MLC_TUNE_TVM_MODEL', '') != '': - print("The \"tune-model\" variation is selected, but at the same time the path to the existing \"work_dir\" is also specified. The compiled model will be based on the found existing \"work_dir\".") + logger.warning("The \"tune-model\" variation is selected, but at the same time the path to the existing \"work_dir\" is also specified. The compiled model will be based on the found existing \"work_dir\".") env["MLC_TUNE_TVM_MODEL"] = "no" return {'return': 0} diff --git a/script/install-bazel/customize.py b/script/install-bazel/customize.py index a3d230c47..08914d8b5 100644 --- a/script/install-bazel/customize.py +++ b/script/install-bazel/customize.py @@ -10,6 +10,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] need_version = env.get('MLC_VERSION', '') @@ -17,7 +19,7 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) + logger.info(f"{recursion_spaces} # Requested version: {need_version}") # if 'MLC_GIT_CHECKOUT' not in env: # env['MLC_GIT_CHECKOUT'] = 'releases/gcc-' + need_version diff --git a/script/install-cmake-prebuilt/customize.py b/script/install-cmake-prebuilt/customize.py index ccdc6ebee..ca2f1e367 100644 --- a/script/install-cmake-prebuilt/customize.py +++ b/script/install-cmake-prebuilt/customize.py @@ -11,6 +11,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] need_version = env.get('MLC_VERSION', '') @@ -18,7 +20,7 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) + logger.info(f"{recursion_spaces} # Requested version: {need_version}") version_split = need_version.split(".") while len(version_split) < 3: @@ -66,10 +68,10 @@ def preprocess(i): package_url = 'https://github.com/Kitware/CMake/releases/download/v' + \ need_version + '/' + package_name - print(recursion_spaces + ' # Prepared package URL: {}'.format(package_url)) + logger.info(f"{recursion_spaces} # Prepared package URL: {package_url}") - print('') - print('Downloading from {} ...'.format(package_url)) + logger.info('') + logger.info(f"Downloading from {package_url} ...") r = download_file({ 'url': package_url}) @@ -80,7 +82,7 @@ def preprocess(i): # Check what to do with this file depending on OS if os_info['platform'] == 'windows': - print('Unzipping file {}'.format(filename)) + logger.info(f"Unzipping file {filename}") r = unzip_file({ 'strip_folders': 1, @@ -89,7 +91,7 @@ def preprocess(i): return r if os.path.isfile(filename): - print('Removing file {}'.format(filename)) + logger.info(f"Removing file {filename}") os.remove(filename) path_bin = os.path.join(os.getcwd(), 'bin') diff --git a/script/install-diffusers-from-src/customize.py b/script/install-diffusers-from-src/customize.py index 300164858..f4a90798c 100644 --- a/script/install-diffusers-from-src/customize.py +++ b/script/install-diffusers-from-src/customize.py @@ -16,7 +16,3 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 5d37ab787..dba1f0d56 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] need_version = env.get('MLC_VERSION', '') @@ -20,7 +22,7 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) + logger.info(f"{recursion_spaces} # Requested version: {need_version}") if 'MLC_GIT_CHECKOUT' not in env: env['MLC_GIT_CHECKOUT'] = 'releases/gcc-' + need_version diff --git a/script/install-generic-conda-package/customize.py b/script/install-generic-conda-package/customize.py index 33c2ad57c..865c3ec83 100644 --- a/script/install-generic-conda-package/customize.py +++ b/script/install-generic-conda-package/customize.py @@ -27,8 +27,12 @@ def preprocess(i): def detect_version(i): + logger = i['automation'].logger + # TBD - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) return {'return': 0, 'version': version} diff --git a/script/install-gflags/customize.py b/script/install-gflags/customize.py index c92d1d7af..f43b25276 100644 --- a/script/install-gflags/customize.py +++ b/script/install-gflags/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] need_version = env.get('MLC_VERSION', '') @@ -20,12 +22,6 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) - - return {'return': 0} + logger.info(f"{recursion_spaces} # Requested version: {need_version}") - -def postprocess(i): - inp = i['input'] - env = i['env'] return {'return': 0} diff --git a/script/install-llvm-prebuilt/customize.py b/script/install-llvm-prebuilt/customize.py index 45417febc..d7a84815b 100644 --- a/script/install-llvm-prebuilt/customize.py +++ b/script/install-llvm-prebuilt/customize.py @@ -10,6 +10,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] need_version = env.get('MLC_VERSION', '') @@ -18,7 +20,7 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) + logger.info(f"{recursion_spaces} # Requested version: {need_version}") host_os_bits = env['MLC_HOST_OS_BITS'] @@ -60,12 +62,14 @@ def preprocess(i): package_name = 'LLVM-' + need_version + '-win' + host_os_bits + '.exe' clang_file_name = "clang.exe" - print('') - print('WARNING: Please copy the following path and then paste it') - print(' when LLVM installer asks you about the "Destination Folder":') - print('') - print(os.getcwd()) - print('') + logger.info('') + logger.warning( + 'WARNING: Please copy the following path and then paste it') + logger.info( + ' when LLVM installer asks you about the "Destination Folder":') + logger.info('') + logger.info(fos.getcwd()) + logger.info('') input('Press Enter to continue!') else: @@ -164,10 +168,10 @@ def preprocess(i): package_url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-' + \ need_version + '/' + package_name - print(recursion_spaces + ' # Prepared package URL: {}'.format(package_url)) + logger.info(f"{recursion_spaces} # Prepared package URL: {package_url}") - print('') - print('Downloading from {} ...'.format(package_url)) + logger.info('') + logger.info(f"Downloading from {package_url} ...") cm = automation.action_object diff --git a/script/install-openssl/customize.py b/script/install-openssl/customize.py index 208ca60f6..c188d6634 100644 --- a/script/install-openssl/customize.py +++ b/script/install-openssl/customize.py @@ -6,6 +6,8 @@ def preprocess(i): os_info = i['os_info'] + logger = i['automation'].logger + if os_info['platform'] == 'windows': return {'return': 1, 'error': 'Windows is not supported in this script yet'} @@ -20,28 +22,6 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) - - return {'return': 0} + logger.info(f"{recursion_spaces} # Requested version: {need_version}") - -def postprocess(i): - inp = i['input'] - env = i['env'] - tags = inp['tags'] - tag_list = tags.split(",") - install_path = os.path.join( - os.getcwd(), - 'openssl-' + - env['MLC_VERSION'] + - 'g', - 'install') - path_lib = os.path.join(install_path, 'lib') - if '+LD_LIBRARY_PATH' not in env: - env['+LD_LIBRARY_PATH'] = [] - env['+LD_LIBRARY_PATH'].append(path_lib) - bin_name = "openssl" - path_bin = os.path.join(install_path, 'bin') - env['MLC_OPENSSL_INSTALLED_PATH'] = path_bin - env['MLC_OPENSSL_BIN_WITH_PATH'] = os.path.join(path_bin, bin_name) return {'return': 0} diff --git a/script/install-python-src/customize.py b/script/install-python-src/customize.py index cf5f95f88..37ac091c8 100644 --- a/script/install-python-src/customize.py +++ b/script/install-python-src/customize.py @@ -6,6 +6,8 @@ def preprocess(i): os_info = i['os_info'] + logger = i['automation'].logger + if os_info['platform'] == 'windows': return {'return': 1, 'error': 'Windows is not supported in this script yet'} @@ -20,7 +22,7 @@ def preprocess(i): return {'return': 1, 'error': 'internal problem - MLC_VERSION is not defined in env'} - print(recursion_spaces + ' # Requested version: {}'.format(need_version)) + logger.info(f"{recursion_spaces} # Requested version: {need_version}") path_bin = os.path.join(os.getcwd(), 'install', 'bin') diff --git a/script/install-python-venv/customize.py b/script/install-python-venv/customize.py index 79dd031f0..7929e3c96 100644 --- a/script/install-python-venv/customize.py +++ b/script/install-python-venv/customize.py @@ -15,6 +15,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + recursion_spaces = i['recursion_spaces'] # Add extra tags to python @@ -23,7 +25,7 @@ def preprocess(i): name = env.get('MLC_NAME', '') if not quiet and name == '': - print('') + logger.info('') x = input( 'Enter some tag to describe this virtual env (mlperf-inf,octoml-bench,etc): ') x = x.strip() diff --git a/script/install-rapidjson-from-src/customize.py b/script/install-rapidjson-from-src/customize.py index 300164858..f4a90798c 100644 --- a/script/install-rapidjson-from-src/customize.py +++ b/script/install-rapidjson-from-src/customize.py @@ -16,7 +16,3 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/install-torchvision-from-src/customize.py b/script/install-torchvision-from-src/customize.py index 300164858..f4a90798c 100644 --- a/script/install-torchvision-from-src/customize.py +++ b/script/install-torchvision-from-src/customize.py @@ -16,7 +16,3 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/plug-prebuilt-cudnn-to-cuda/customize.py b/script/plug-prebuilt-cudnn-to-cuda/customize.py index c37d15b17..9c2939a0f 100644 --- a/script/plug-prebuilt-cudnn-to-cuda/customize.py +++ b/script/plug-prebuilt-cudnn-to-cuda/customize.py @@ -18,6 +18,8 @@ def preprocess(i): meta = i['meta'] automation = i['automation'] + logger = automation.logger + version = env.get('MLC_VERSION') supported_versions = list(meta['versions'].keys()) @@ -42,8 +44,8 @@ def preprocess(i): cudnn_url = f'https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/{filename}' - print('') - print(f'URL to download cuDNN: {cudnn_url}') + logger.info('') + logger.info(f'URL to download cuDNN: {cudnn_url}') env['MLC_CUDNN_TAR_DIR'] = cudnn_dir env['MLC_CUDNN_UNTAR_PATH'] = os.path.join(cur_dir, cudnn_dir) diff --git a/script/plug-prebuilt-cusparselt-to-cuda/customize.py b/script/plug-prebuilt-cusparselt-to-cuda/customize.py index ec679ac7d..d5564c0a8 100644 --- a/script/plug-prebuilt-cusparselt-to-cuda/customize.py +++ b/script/plug-prebuilt-cusparselt-to-cuda/customize.py @@ -18,6 +18,8 @@ def preprocess(i): meta = i['meta'] automation = i['automation'] + logger = automation.logger + version = env.get('MLC_VERSION') supported_versions = list(meta['versions'].keys()) @@ -42,8 +44,8 @@ def preprocess(i): cusparselt_url = f'https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/{filename}' - print('') - print(f'URL to download CUSPARSELT: {cusparselt_url}') + logger.info('') + logger.info(f'URL to download CUSPARSELT: {cusparselt_url}') env['MLC_CUSPARSELT_TAR_DIR'] = cusparselt_dir env['MLC_CUSPARSELT_UNTAR_PATH'] = os.path.join(cur_dir, cusparselt_dir) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index a394b4446..1db5d28e9 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -9,15 +9,18 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger + submission_dir = env.get("MLC_MLPERF_INFERENCE_SUBMISSION_DIR", "") if submission_dir == "": - print("Please set --env.MLC_MLPERF_INFERENCE_SUBMISSION_DIR") + logger.error("Please set --env.MLC_MLPERF_INFERENCE_SUBMISSION_DIR") return {'return': 1, 'error': 'MLC_MLPERF_INFERENCE_SUBMISSION_DIR is not specified'} if not os.path.exists(submission_dir): - print("Please set --env.MLC_MLPERF_INFERENCE_SUBMISSION_DIR to a valid submission directory") + logger.error( + "Please set --env.MLC_MLPERF_INFERENCE_SUBMISSION_DIR to a valid submission directory") return {'return': 1, 'error': 'MLC_MLPERF_INFERENCE_SUBMISSION_DIR is not existing'} @@ -26,7 +29,7 @@ def preprocess(i): submission_processed = f"{submission_dir}_processed" if os.path.exists(submission_processed): - print(f"Cleaning {submission_processed}") + logger.info(f"Cleaning {submission_processed}") shutil.rmtree(submission_processed) version = env.get('MLC_MLPERF_SUBMISSION_CHECKER_VERSION', '') diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 967fde0b5..4ef3fbd43 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -9,10 +9,12 @@ def preprocess(i): xsep = ';' if os_info['platform'] == 'windows' else ':' env = i['env'] + logger = i['automation'].logger + results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") if results_dir == "": - print("Please set MLC_MLPERF_ACCURACY_RESULTS_DIR") + logger.error("Please set MLC_MLPERF_ACCURACY_RESULTS_DIR") return {'return': -1} # In fact, we expect only 1 command line here @@ -229,7 +231,7 @@ def postprocess(i): os_info = i['os_info'] env = i['env'] state = i['state'] - + logger = i['automation'].logger xsep = ';' if os_info['platform'] == 'windows' else ':' results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") @@ -240,16 +242,16 @@ def postprocess(i): accuracy_file = os.path.join(result_dir, "accuracy.txt") if os.path.exists(accuracy_file): - print('') - print('Accuracy file: {}'.format(accuracy_file)) - print('') + logger.info('') + logger.info('Accuracy file: {}'.format(accuracy_file)) + logger.info('') x = '' with open(accuracy_file, "r") as fp: x = fp.read() if x != '': - print(x) + logger.info(f"{x}") # Trying to extract accuracy dict for y in x.split('\n'): @@ -265,5 +267,5 @@ def postprocess(i): except ValueError as e: pass - print('') + logger.info('') return {'return': 0} diff --git a/script/prune-bert-models/customize.py b/script/prune-bert-models/customize.py index 9928625f5..e69cde689 100644 --- a/script/prune-bert-models/customize.py +++ b/script/prune-bert-models/customize.py @@ -8,6 +8,8 @@ def preprocess(i): env = i['env'] + logger = i['automation'].logger + ckpt_path = env.get('MLC_BERT_PRUNE_CKPT_PATH', '') if ckpt_path == '': p = env['MLC_ML_MODEL_FILE_WITH_PATH'] @@ -29,17 +31,17 @@ def preprocess(i): out_dir = os.path.join(os.getcwd(), 'pruned-model-output') env['MLC_BERT_PRUNE_OUTPUT_DIR'] = out_dir - print('') + logger.info('') print( 'Local CM cache path to the updated BERT pruner src from NeurIPS 2022: ' + env['MLC_GIT_REPO_BERT_PRUNER_NEURIPS_2022_CHECKOUT_PATH']) - print('') + logger.info('') for k in ["MLC_ML_MODEL_FILE_WITH_PATH", "MLC_BERT_PRUNE_CKPT_PATH", "MLC_BERT_PRUNE_OUTPUT_DIR"]: - print('ENV["{}"]: {}'.format(k, env[k])) + logger.info(f"ENV[\"{k}\"]: {env[k]}") - print('') + logger.info('') return {'return': 0} @@ -48,6 +50,8 @@ def postprocess(i): env = i['env'] - print("Entered postprocess") + logger = i['automation'].logger + + logger.info("Entered postprocess") return {'return': 0} diff --git a/script/push-csv-to-spreadsheet/customize.py b/script/push-csv-to-spreadsheet/customize.py index e3a082660..1326b6edf 100644 --- a/script/push-csv-to-spreadsheet/customize.py +++ b/script/push-csv-to-spreadsheet/customize.py @@ -10,7 +10,3 @@ def preprocess(i): automation = i['automation'] return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/push-mlperf-inference-results-to-github/customize.py b/script/push-mlperf-inference-results-to-github/customize.py index d056eec77..9b2df4d0f 100644 --- a/script/push-mlperf-inference-results-to-github/customize.py +++ b/script/push-mlperf-inference-results-to-github/customize.py @@ -42,7 +42,3 @@ def preprocess(i): env['MLC_GIT_PUSH_CMD'] = "git push" return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/run-all-mlperf-models/customize.py b/script/run-all-mlperf-models/customize.py index 9bfdc51f9..c9b6f68fa 100644 --- a/script/run-all-mlperf-models/customize.py +++ b/script/run-all-mlperf-models/customize.py @@ -13,7 +13,7 @@ def preprocess(i): script_path = i['run_script_input']['path'] automation = i['automation'] - + logger = automation.logger quiet = is_true(env.get('MLC_QUIET', False)) models = env['MODELS'].split(",") @@ -26,7 +26,7 @@ def preprocess(i): if devices: devices = devices.split(",") - print(backends) + logger.info(f"BACKENDS: {backends}") implementation = env['IMPLEMENTATION'] power = env.get('POWER', '') @@ -101,7 +101,7 @@ def preprocess(i): run_script_content += "\n\n" + "\n\n".join(cmds) with open(os.path.join(script_path, run_file_name + ".sh"), 'w') as f: f.write(run_script_content) - print(cmds) + logger.info(f"CMDS: {cmds}") return {'return': 0} diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index c636f4fa7..62eb7f535 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -14,6 +14,8 @@ def preprocess(i): mlc = i['automation'].action_object + logger = i['automation'].logger + interactive = env.get('MLC_DOCKER_INTERACTIVE_MODE', '') if is_true(interactive): @@ -51,9 +53,9 @@ def preprocess(i): DOCKER_CONTAINER = docker_image_repo + "/" + \ docker_image_name + ":" + docker_image_tag - print('') - print('Checking existing Docker container:') - print('') + logger.info('') + logger.info('Checking existing Docker container:') + logger.info('') # CMD = f"""{env['MLC_CONTAINER_TOOL']} ps --format=json --filter "ancestor={DOCKER_CONTAINER}" """ CMD = f"""{env['MLC_CONTAINER_TOOL']} ps --format """ + \ '"{{ .ID }},"' + f""" --filter "ancestor={DOCKER_CONTAINER}" """ @@ -61,8 +63,8 @@ def preprocess(i): CMD += " 2> nul" else: CMD += " 2> /dev/null" - print(' ' + CMD) - print('') + logger.info(' ' + CMD) + logger.info('') try: out = subprocess.check_output( @@ -81,7 +83,7 @@ def preprocess(i): if existing_container_id and is_true( env.get('MLC_DOCKER_REUSE_EXISTING_CONTAINER', '')): - print(f"Reusing existing container {existing_container_id}") + logger.info(f"Reusing existing container {existing_container_id}") env['MLC_DOCKER_CONTAINER_ID'] = existing_container_id else: @@ -89,7 +91,7 @@ def preprocess(i): print( f"""Not using existing container {existing_container_id} as env['MLC_DOCKER_REUSE_EXISTING_CONTAINER'] is not set""") else: - print("No existing container") + logger.info("No existing container") if env.get('MLC_DOCKER_CONTAINER_ID', '') != '': del (env['MLC_DOCKER_CONTAINER_ID']) # not valid ID @@ -100,11 +102,11 @@ def preprocess(i): else: CMD += " 2> /dev/null" - print('') - print('Checking Docker images:') - print('') - print(' ' + CMD) - print('') + logger.info('') + logger.info('Checking Docker images:') + logger.info('') + logger.info(' ' + CMD) + logger.info('') try: docker_image = subprocess.check_output( @@ -117,7 +119,7 @@ def preprocess(i): if is_false(recreate_image): if docker_image: - print("Docker image exists with ID: " + docker_image) + logger.info("Docker image exists with ID: " + docker_image) env['MLC_DOCKER_IMAGE_EXISTS'] = "yes" # elif recreate_image == "yes": @@ -131,6 +133,8 @@ def postprocess(i): env = i['env'] + logger = i['automation'].logger + # Updating Docker info update_docker_info(env) @@ -258,11 +262,11 @@ def postprocess(i): CMD += ' && echo "ID=$ID"' - print('=========================') - print("Container launch command:") - print('') - print(CMD) - print('') + logger.info('=========================') + logger.info("Container launch command:") + logger.info('') + logger.info(f"{CMD}") + logger.info('') print( "Running " + run_cmd + @@ -270,7 +274,7 @@ def postprocess(i): record_script({'cmd': CMD, 'env': env}) - print('') + logger.info('') # Execute the command try: result = subprocess.run( @@ -280,12 +284,12 @@ def postprocess(i): stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - print("Command Output:", result.stdout) + logger.info("Command Output:", result.stdout) except subprocess.CalledProcessError as e: - print("Error Occurred!") - print(f"Command: {e.cmd}") - print(f"Return Code: {e.returncode}") - print(f"Error Output: {e.stderr}") + logger.error("Error Occurred!") + logger.info(f"Command: {e.cmd}") + logger.info(f"Return Code: {e.returncode}") + logger.error(f"Error Output: {e.stderr}") return {'return': 1, 'error': e.stderr} docker_out = result.stdout @@ -300,7 +304,7 @@ def postprocess(i): ID = line[3:] env['MLC_DOCKER_CONTAINER_ID'] = ID - print(docker_out) + logger.info(f"{docker_out}") else: x = "'" @@ -319,14 +323,14 @@ def postprocess(i): " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag CMD = CONTAINER + " bash -c " + x + run_cmd_prefix + run_cmd + x2 + x - print('') - print("Container launch command:") - print('') - print(CMD) + logger.info('') + logger.info("Container launch command:") + logger.info('') + logger.info(f"{CMD}") record_script({'cmd': CMD, 'env': env}) - print('') + logger.info('') docker_out = os.system(CMD) if docker_out != 0: if docker_out % 256 == 0: diff --git a/script/run-mlperf-automotive-app/customize.py b/script/run-mlperf-automotive-app/customize.py index 9309390a6..c3dda9c7a 100644 --- a/script/run-mlperf-automotive-app/customize.py +++ b/script/run-mlperf-automotive-app/customize.py @@ -21,6 +21,7 @@ def preprocess(i): state = i['state'] script_path = i['run_script_input']['path'] mlc = i['automation'].action_object + logger = i['automation'].logger if is_true(env.get('MLC_RUN_DOCKER_CONTAINER', '')): return {'return': 0} @@ -191,12 +192,12 @@ def preprocess(i): if clean: path_to_clean = output_dir - print('=========================================================') - print('Cleaning results in {}'.format(path_to_clean)) + logger.info('=========================================================') + logger.info('Cleaning results in {}'.format(path_to_clean)) if os.path.exists(path_to_clean): shutil.rmtree(path_to_clean) - print('=========================================================') + logger.info('=========================================================') if is_true(str(env.get('MLC_MLPERF_USE_DOCKER', ''))): action = "docker" @@ -242,7 +243,8 @@ def preprocess(i): "MLC_TMP_"): del env_copy[key] - print(f"\nRunning loadgen scenario: {scenario} and mode: {mode}") + logger.info( + f"\nRunning loadgen scenario: {scenario} and mode: {mode}") ii = {'action': action, 'automation': 'script', 'tags': scenario_tags, 'quiet': 'true', 'env': env_copy, 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': copy.deepcopy(add_deps_recursive), 'ad': ad, 'adr': copy.deepcopy(adr), 'v': verbose, 'print_env': print_env, 'print_deps': print_deps, 'dump_version_info': dump_version_info} @@ -293,10 +295,11 @@ def preprocess(i): # executing CM import mlperf_utils # noqa - print(sut) + logger.info(f"{sut}") result_table, headers = mlperf_utils.get_result_table( state["mlc-mlperf-inference-results"][sut]) - print(tabulate(result_table, headers=headers, tablefmt="pretty")) + logger.info( + f"{tabulate(result_table, headers=headers, tablefmt='pretty')}") print( f"\nThe MLPerf inference results are stored at {output_dir}\n") @@ -348,18 +351,18 @@ def postprocess(i): env = i['env'] state = i['state'] - + logger = i['automation'].logger if env.get('MLC_MLPERF_IMPLEMENTATION', '') == 'reference': x1 = env.get('MLC_MLPERF_INFERENCE_SOURCE', '') x2 = env.get('MLC_MLPERF_INFERENCE_CONF_PATH', '') if x1 != '' and x2 != '': - print('') + logger.info('') print( 'Path to the MLPerf inference benchmark reference sources: {}'.format(x1)) print( 'Path to the MLPerf inference reference configuration file: {}'.format(x2)) - print('') + logger.info('') return {'return': 0} diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 2e9c279d2..8cf0fbe1c 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -21,7 +21,7 @@ def preprocess(i): inp = i['input'] state = i['state'] script_path = i['run_script_input']['path'] - + logger = i['automation'].logger if is_true(env.get('MLC_RUN_DOCKER_CONTAINER', '')): return {'return': 0} @@ -215,12 +215,12 @@ def preprocess(i): if clean: path_to_clean = output_dir - print('=========================================================') - print('Cleaning results in {}'.format(path_to_clean)) + logger.info('=========================================================') + logger.info('Cleaning results in {}'.format(path_to_clean)) if os.path.exists(path_to_clean): shutil.rmtree(path_to_clean) - print('=========================================================') + logger.info('=========================================================') if is_true(env.get('MLC_MLPERF_USE_DOCKER', '')): action = "docker" @@ -270,7 +270,8 @@ def preprocess(i): env_copy = copy.deepcopy(env) const_copy = copy.deepcopy(const) - print(f"\nRunning loadgen scenario: {scenario} and mode: {mode}") + logger.info( + f"\nRunning loadgen scenario: {scenario} and mode: {mode}") ii = {'action': action, 'automation': 'script', 'tags': scenario_tags, 'quiet': 'true', 'env': env_copy, 'const': const_copy, 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': copy.deepcopy(add_deps_recursive), 'ad': ad, 'adr': copy.deepcopy(adr), 'print_env': print_env, 'print_deps': print_deps, 'dump_version_info': dump_version_info} @@ -335,10 +336,11 @@ def preprocess(i): # executing CM from tabulate import tabulate # noqa - print(sut) + logger.info(f"{sut}") result_table, headers = mlperf_utils.get_result_table( state["mlc-mlperf-inference-results"][sut]) - print(tabulate(result_table, headers=headers, tablefmt="pretty")) + logger.info( + f"{tabulate(result_table, headers=headers, tablefmt='pretty')}") print( f"\nThe MLPerf inference results are stored at {output_dir}\n") @@ -390,18 +392,18 @@ def postprocess(i): env = i['env'] state = i['state'] - + logger = i['automation'].logger if env.get('MLC_MLPERF_IMPLEMENTATION', '') == 'reference': x1 = env.get('MLC_MLPERF_INFERENCE_SOURCE', '') x2 = env.get('MLC_MLPERF_INFERENCE_CONF_PATH', '') if x1 != '' and x2 != '': - print('') + logger.info('') print( 'Path to the MLPerf inference benchmark reference sources: {}'.format(x1)) print( 'Path to the MLPerf inference reference configuration file: {}'.format(x2)) - print('') + logger.info('') return {'return': 0} diff --git a/script/run-mlperf-inference-mobilenet-models/customize.py b/script/run-mlperf-inference-mobilenet-models/customize.py index ee9ba88d8..341efd37d 100644 --- a/script/run-mlperf-inference-mobilenet-models/customize.py +++ b/script/run-mlperf-inference-mobilenet-models/customize.py @@ -19,6 +19,7 @@ def preprocess(i): adr = i['input'].get('adr') automation = i['automation'] + logger = automation.logger # mlc = i['automation'].action_object # cache_action = i['automation'].cache_action cache_action = mlc @@ -189,7 +190,7 @@ def preprocess(i): if is_true(env.get('MLC_MLPERF_POWER', '')): mlc_input['power'] = 'yes' - print(mlc_input) + logger.info(f"{mlc_input}") r = mlc.access(mlc_input) if r['return'] > 0: return r @@ -198,7 +199,7 @@ def preprocess(i): if is_true(env.get('MLC_MINIMIZE_DISK_USAGE', '')): r = cache_action.access(clean_input) if r['return'] > 0: - print(r) + logger.info(f"{r}") # return r else: importlib.reload(mlc.action) @@ -209,7 +210,7 @@ def preprocess(i): ''' r = cache_action.access(clean_input) if r['return'] > 0: - print(r) + logger.info(fr) # return r else: importlib.reload(mlc.action) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 6dcd5e1cf..a23bf2947 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -8,6 +8,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger q = '"' if os_info['platform'] == 'windows' else "'" submission_dir = env.get("MLC_MLPERF_INFERENCE_SUBMISSION_DIR", "") @@ -84,7 +85,7 @@ def preprocess(i): report_generator_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "tools", "submission", "generate_final_report.py") env['MLC_RUN_CMD'] = CMD - print(CMD) + logger.info(f"{CMD}") env['MLC_POST_RUN_CMD'] = env['MLC_PYTHON_BIN_WITH_PATH'] + ' ' + q + report_generator_file + q + ' --input summary.csv ' + \ x_version + \ x_submission_repo_name + \ diff --git a/script/run-mlperf-power-client/customize.py b/script/run-mlperf-power-client/customize.py index f7d3cd989..0c2b9fbcd 100644 --- a/script/run-mlperf-power-client/customize.py +++ b/script/run-mlperf-power-client/customize.py @@ -40,7 +40,3 @@ def preprocess(i): env['MLC_MLPERF_POWER_RUN_CMD'] = cmd return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/run-mlperf-power-server/customize.py b/script/run-mlperf-power-server/customize.py index e4117c93d..59056a0d2 100644 --- a/script/run-mlperf-power-server/customize.py +++ b/script/run-mlperf-power-server/customize.py @@ -7,7 +7,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - + logger = i['automation'].logger # Initialize ConfigParser config = configparser.ConfigParser() @@ -66,7 +66,8 @@ def preprocess(i): with open('tmp-power-server.conf', 'w') as configfile: config.write(configfile) - print({section: dict(config[section]) for section in config.sections()}) + logger.info( + f"{section: dict(config[section]) for section in config.sections()}") if env['MLC_HOST_OS_TYPE'] == "windows": cmd_prefix = "" @@ -85,7 +86,3 @@ def preprocess(i): env['RUN_CMD'] = cmd return {'return': 0} - - -def postprocess(i): - return {'return': 0} diff --git a/script/run-terraform/customize.py b/script/run-terraform/customize.py index 15687e6a9..d90962d03 100644 --- a/script/run-terraform/customize.py +++ b/script/run-terraform/customize.py @@ -8,6 +8,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger script_dir = i['run_script_input']['path'] config_dir = os.path.join( script_dir, env.get( @@ -15,7 +16,7 @@ def preprocess(i): env['MLC_TERRAFORM_CONFIG_DIR'] = config_dir cache_dir = os.getcwd() - print(f"Running terraform from {cache_dir}") + logger.info(f"Running terraform from {cache_dir}") shutil.copy(os.path.join(config_dir, "main.tf"), cache_dir) env['MLC_TERRAFORM_RUN_DIR'] = cache_dir @@ -24,6 +25,7 @@ def preprocess(i): def postprocess(i): + logger = i["automation"].logger env = i['env'] if env.get('MLC_DESTROY_TERRAFORM'): return {'return': 0} @@ -83,8 +85,3 @@ def postprocess(i): print_attr(instance_attributes, "security_groups") return {'return': 0} - - -def print_attr(instance_attributes, key): - if key in instance_attributes: - print(key.upper() + ": " + str(instance_attributes[key])) diff --git a/script/run-vllm-server/customize.py b/script/run-vllm-server/customize.py index 68eceb84d..b82e3219a 100644 --- a/script/run-vllm-server/customize.py +++ b/script/run-vllm-server/customize.py @@ -10,8 +10,11 @@ def preprocess(i): meta = i['meta'] automation = i['automation'] - # Required parameter check - model_name = env.get("MLC_VLLM_SERVER_MODEL_NAME") + logger = automation.logger + + cmd_args = "" + + model_name = env.get("MLC_VLLM_SERVER_MODEL_NAME", False) if not model_name: return {'return': 1, 'error': 'Model name not specified'} @@ -93,7 +96,7 @@ def preprocess(i): cmd_args += f" {arg_name}" cmd = f"{env['MLC_PYTHON_BIN_WITH_PATH']} -m vllm.entrypoints.openai.api_server {cmd_args}" - print(cmd) + logger.info(f"{cmd}") env['MLC_VLLM_RUN_CMD'] = cmd diff --git a/script/runtime-system-infos/customize.py b/script/runtime-system-infos/customize.py index b54abbdac..ae4d7d61b 100644 --- a/script/runtime-system-infos/customize.py +++ b/script/runtime-system-infos/customize.py @@ -36,6 +36,7 @@ def preprocess(i): return {'return': 1, 'error': 'Windows is not supported in this script yet'} env = i['env'] + logger = i['automation'].logger if env.get("MLC_RUN_DIR", "") == "": env['MLC_RUN_DIR'] = os.getcwd() @@ -46,11 +47,12 @@ def preprocess(i): interval = int(env.get('MLC_SYSTEM_INFO_MEASUREMENT_INTERVAL', '2')) - print(f"The system dumps are created to the folder:{logs_dir}") + logger.info(f"The system dumps are created to the folder:{logs_dir}") - print("WARNING: Currently the script is in its development stage. Only memory measurements supports as of now!") + logger.warning( + "Currently the script is in its development stage. Only memory measurements supports as of now!") - print("Started measuring system info!") + logger.info("Started measuring system info!") csv_headers = [ 'timestamp', diff --git a/script/set-venv/customize.py b/script/set-venv/customize.py index f1e0f726b..daf23ea4f 100644 --- a/script/set-venv/customize.py +++ b/script/set-venv/customize.py @@ -13,6 +13,8 @@ def preprocess(i): automation = i['automation'] + logger = automation.logger + quiet = is_true(env.get('MLC_QUIET', False)) ############################################################ @@ -49,9 +51,10 @@ def preprocess(i): cmd = python_path + ' -m venv ' + name + create_dir.format(name) - print('====================================================================') + logger.info( + '====================================================================') - print('Creating venv: "{}" ...'.format(cmd)) + logger.info('Creating venv: "{}" ...'.format(cmd)) os.system(cmd) if os.path.isfile(activate_script2): @@ -86,11 +89,13 @@ def preprocess(i): with open(script_file, 'w') as f: f.write(cmd) - print('====================================================================') - print('Please run the following command:') - print('') - print(xcmd) - print('====================================================================') + logger.info( + '====================================================================') + logger.info('Please run the following command:') + logger.info('') + logger.info(f"{xcmd}") + logger.info( + '====================================================================') return {'return': 0} diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index ae5824af2..db89c4655 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -10,7 +10,7 @@ def preprocess(i): env = i['env'] meta = i['meta'] automation = i['automation'] - + logger = automation.logger server = env['MLC_MLPERF_SUBMISSION_URL'] benchmark = env['MLC_MLPERF_BENCHMARK'] submitter_id = env['MLC_MLPERF_SUBMITTER_ID'] @@ -39,12 +39,12 @@ def preprocess(i): # print(signed_url) # print(submission_id) - r = upload_file_to_signed_url(file_path, signed_url) + r = upload_file_to_signed_url(file_path, signed_url, logger) if r['return'] > 0: return r r = trigger_submission_checker( - server, submitter_id, benchmark, submission_id) + server, submitter_id, benchmark, submission_id, logger) if r['return'] > 0: return r @@ -103,7 +103,7 @@ def get_signed_url(server, benchmark, submitter_id, submitter_name, file_path): 'submission_id': submission_id} -def upload_file_to_signed_url(file_path, signed_url): +def upload_file_to_signed_url(file_path, signed_url, logger): """ Uploads a file to a signed URL using HTTP PUT. @@ -129,14 +129,14 @@ def upload_file_to_signed_url(file_path, signed_url): ) if response.status_code in [200, 201, 204]: - print("File uploaded successfully!") + logger.info("File uploaded successfully!") return { 'return': 0 } else: print( f"Failed to upload file. Status code: {response.status_code}") - print("Response:", response.text) + logger.info("Response:", response.text) return { 'return': response.status_code, @@ -144,14 +144,14 @@ def upload_file_to_signed_url(file_path, signed_url): } except FileNotFoundError: - print("Error: File not found.") + logger.error("Error: File not found.") return { 'return': 400, 'error': f'''File {file_path} not found''' } except requests.exceptions.RequestException as e: - print(f"Request failed: {e}") + logger.error(f"Request failed: {e}") return { 'return': 500, 'error': str(e) @@ -159,7 +159,7 @@ def upload_file_to_signed_url(file_path, signed_url): def trigger_submission_checker( - server_url, submitter_id, benchmark, submission_id): + server_url, submitter_id, benchmark, submission_id, logger): """ Sends a POST request with URL-encoded form data. @@ -187,12 +187,12 @@ def trigger_submission_checker( response = requests.post(url, data=payload, headers=headers) if response.ok: - print("Submission Check Request successful!") + logger.info("Submission Check Request successful!") pass else: - print( + logger.error( f"Submission Check Request failed with status code: {response.status_code}") - print("Response:", response.text) + logger.error("Response:", response.text) return { "return": 0, @@ -200,12 +200,8 @@ def trigger_submission_checker( } except requests.exceptions.RequestException as e: - print("An error occurred:", e) + logger.error("An error occurred:", e) return { "return": 500, "error": str(e) } - - -def postprocess(i): - return {'return': 0} diff --git a/script/tar-my-folder/customize.py b/script/tar-my-folder/customize.py index a8407ff2c..0ce726cdd 100644 --- a/script/tar-my-folder/customize.py +++ b/script/tar-my-folder/customize.py @@ -8,6 +8,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger input_dir = env.get("MLC_TAR_INPUT_DIR", "") if input_dir == "": return {'return': 1, 'error': 'Please set MLC_TAR_INPUT_DIR'} @@ -39,8 +40,8 @@ def preprocess(i): str(cd_dir) + ' -czf ' + os.path.join(output_dir, output_file) + ' ' + input_dirname - print(CMD) + logger.info(f"{CMD}") ret = os.system(CMD) - print("Tar file " + os.path.join(output_dir, output_file) + " created") + logger.info(f"Tar file {os.path.join(output_dir, output_file)} created") return {'return': ret} diff --git a/script/truncate-mlperf-inference-accuracy-log/customize.py b/script/truncate-mlperf-inference-accuracy-log/customize.py index 3e609188f..abe20dcdb 100644 --- a/script/truncate-mlperf-inference-accuracy-log/customize.py +++ b/script/truncate-mlperf-inference-accuracy-log/customize.py @@ -8,10 +8,11 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + logger = i['automation'].logger submission_dir = env.get("MLC_MLPERF_INFERENCE_SUBMISSION_DIR", "") if submission_dir == "": - print("Please set MLC_MLPERF_INFERENCE_SUBMISSION_DIR") + logger.error("Please set MLC_MLPERF_INFERENCE_SUBMISSION_DIR") return {'return': 1, 'error': 'MLC_MLPERF_INFERENCE_SUBMISSION_DIR is not specified in env in run-mlperf-accuracy-log-truncator'} submitter = env.get("MLC_MLPERF_SUBMITTER", "CTuning") diff --git a/script/wrapper-reproduce-octoml-tinyml-submission/customize.py b/script/wrapper-reproduce-octoml-tinyml-submission/customize.py index 27907b617..dd290e09d 100644 --- a/script/wrapper-reproduce-octoml-tinyml-submission/customize.py +++ b/script/wrapper-reproduce-octoml-tinyml-submission/customize.py @@ -37,7 +37,3 @@ def preprocess(i): return r return {'return': 0} - - -def postprocess(i): - return {'return': 0} From c5bfa55c6f002788c4e872c6357d39f29e316e70 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Apr 2025 16:01:54 +0100 Subject: [PATCH 51/55] Use num_threads=1 for retinanet (#397) --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 8d732b413..130fead1b 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -123,6 +123,8 @@ def preprocess(i): if int( NUM_THREADS) > 2 and env['MLC_MLPERF_DEVICE'] == "gpu" and env['MLC_MODEL'] != "rgat": NUM_THREADS = "2" # Don't use more than 2 threads when run on GPU + if env['MLC_MODEL'] in ['retinanet']: + NUM_THREADS = "1" if env['MLC_MODEL'] in ['resnet50', 'retinanet', 'stable-diffusion-xl', 'rgat']: From 7e3d65e0e55e9b33d229a4fa1917779a76cefbb8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:30:40 +0530 Subject: [PATCH 52/55] added experiment to script automation (#398) --- automation/script/module.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/automation/script/module.py b/automation/script/module.py index 28fe3e978..7cbec2196 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4469,6 +4469,13 @@ def docker(self, i): from script.docker import docker_run return docker_run(self, i) + ############################################################ + # portion for experiment action. + # as of now, the experiment action directly calls the run action. + # in the future, we will add more functionality to the experiment action. + def experiment(self, i): + return self.run(i) + ########################################################################## def _available_variations(self, i): From 06b95fa9f0b3e5cedf5295a7b630442b2f9ffac3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 25 Apr 2025 09:01:01 +0000 Subject: [PATCH 53/55] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 7cbec2196..2c728cb49 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4470,7 +4470,7 @@ def docker(self, i): return docker_run(self, i) ############################################################ - # portion for experiment action. + # portion for experiment action. # as of now, the experiment action directly calls the run action. # in the future, we will add more functionality to the experiment action. def experiment(self, i): From 9fba422211efdf729aae8eeeef05a565d189fb19 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 6 May 2025 20:36:49 +0530 Subject: [PATCH 54/55] add --multi-thread-streams=0 for rclone version >= 1.60.0 (#402) --- script/download-file/customize.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 6200cc223..43c579346 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -234,6 +234,17 @@ def preprocess(i): pre_clean = False if not verify_ssl: extra_download_options += " --no-check-certificate " + # rlone fix : check rclone version to add --multi-thread-streams=0 + tmp_rclone_version = env.get('MLC_RCLONE_VERSION', '') + if tmp_rclone_version == '': + logger.warning( + "MLC_RCLONE_VERSION not set, was get-rclone called as dependency?" + ) + else: + ref_version = list(map(int, "1.60.0".split('.'))) + rclone_version = list(map(int, tmp_rclone_version.split('.'))) + if rclone_version >= ref_version: + extra_download_options += " --multi-thread-streams=0 " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is # going to be used anywhere after this point From 086a7a5470ddb42dd8ac6b9f56468f36c2a3fb13 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 7 May 2025 23:33:02 +0100 Subject: [PATCH 55/55] Fixes for llvm-install-src (#404) * Support _path for install,llvm --- automation/script/module.py | 2 +- script/get-llvm/customize.py | 2 +- script/install-llvm-src/customize.py | 54 ++++++++++++++++------------ script/install-llvm-src/meta.yaml | 10 ++++++ 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 2c728cb49..bb1f73440 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3188,7 +3188,7 @@ def _update_variation_meta_with_dynamic_suffix( item_value[i] = l_item.replace( "#", variation_tag_dynamic_suffix) else: - value[item] = value[item].replace( + value[item] = str(value[item]).replace( "#", variation_tag_dynamic_suffix) else: # scalar value, never used? diff --git a/script/get-llvm/customize.py b/script/get-llvm/customize.py index 070f8d436..05356cf0c 100644 --- a/script/get-llvm/customize.py +++ b/script/get-llvm/customize.py @@ -21,7 +21,7 @@ def preprocess(i): if os.path.exists(os.path.join(llvm_path, 'bin', 'clang')): env['MLC_TMP_PATH'] = os.path.join(llvm_path, 'bin') else: - for l in os.listdir(aocc_path): + for l in os.listdir(llvm_path): if os.path.exists(os.path.join( llvm_path, l, 'bin', 'clang')): llvm_path = os.path.join(llvm_path, l) diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 66ae5ea87..2f0c25412 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -17,36 +17,44 @@ def preprocess(i): clang_file_name = "clang" extra_cmake_options = '' - install_prefix = os.path.join(os.getcwd(), "install") - - if env.get('MLC_LLVM_CONDA_ENV', '') == "yes": - install_prefix = env['MLC_CONDA_PREFIX'] - extra_cmake_options = f"-DCMAKE_SHARED_LINKER_FLAGS=-L{install_prefix} -Wl,-rpath,{install_prefix}" - - if is_true(env.get('MLC_LLVM_16_INTEL_MLPERF_INFERENCE', '')): - env['MLC_REQUIRE_INSTALL'] = 'yes' - i['run_script_input']['script_name'] = "install-llvm-16-intel-mlperf-inference" - clang_file_name = "llvm-link" - # env['USE_LLVM'] = install_prefix - # env['LLVM_DIR'] = os.path.join(env['USE_LLVM'], "lib", "cmake", "llvm") + if env.get('MLC_LLVM_INSTALLED_PATH', '') != '' and os.path.exists( + env.get('MLC_LLVM_INSTALLED_PATH')): + install_prefix = env['MLC_LLVM_INSTALLED_PATH'] else: - if env.get('+MLC_LLVM_ENABLE_RUNTIMES', '') != '': - enable_runtimes = ";".join(env['+MLC_LLVM_ENABLE_RUNTIMES']) - else: - enable_runtimes = '' + install_prefix = os.path.join(os.getcwd(), "install") - if env.get('+MLC_LLVM_ENABLE_PROJECTS', '') != '': - enable_projects = ";".join(env['+MLC_LLVM_ENABLE_PROJECTS']) + if os.path.exists(os.path.join(install_prefix, "bin", "clang")) and is_true( + env.get('MLC_LLVM_USE_INSTALLED_DIR')): + i['run_script_input']['script_name'] = "no-install" # skip install + else: + if env.get('MLC_LLVM_CONDA_ENV', '') == "yes": + install_prefix = env['MLC_CONDA_PREFIX'] + extra_cmake_options = f"-DCMAKE_SHARED_LINKER_FLAGS=-L{install_prefix} -Wl,-rpath,{install_prefix}" + + if is_true(env.get('MLC_LLVM_16_INTEL_MLPERF_INFERENCE', '')): + env['MLC_REQUIRE_INSTALL'] = 'yes' + i['run_script_input']['script_name'] = "install-llvm-16-intel-mlperf-inference" + clang_file_name = "llvm-link" + # env['USE_LLVM'] = install_prefix + # env['LLVM_DIR'] = os.path.join(env['USE_LLVM'], "lib", "cmake", "llvm") else: - enable_projects = '' + if env.get('+MLC_LLVM_ENABLE_RUNTIMES', '') != '': + enable_runtimes = ";".join(env['+MLC_LLVM_ENABLE_RUNTIMES']) + else: + enable_runtimes = '' + + if env.get('+MLC_LLVM_ENABLE_PROJECTS', '') != '': + enable_projects = ";".join(env['+MLC_LLVM_ENABLE_PROJECTS']) + else: + enable_projects = '' - llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] + llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] - targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') + targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') - cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" - env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd + env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd need_version = env.get('MLC_VERSION', '') diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 2d22a0446..c59f0a5c2 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -22,6 +22,9 @@ deps: names: - llvm-src-repo tags: get,git,repo + skip_if_env: + MLC_LLVM_INSTALLED_PATH: + - on update_tags_from_env_with_prefix: _branch.: - MLC_GIT_CHECKOUT @@ -46,6 +49,9 @@ post_deps: MLC_REQUIRE_INSTALL: - 'yes' tags: get,llvm + update_tags_from_env_with_prefix: + _path.: + - MLC_LLVM_INSTALLED_PATH prehook_deps: [] sort: 1000 tags: @@ -97,6 +103,10 @@ variations: env: MLC_LLVM_BUILD_TYPE: debug group: build-type + path.#: + env: + MLC_LLVM_INSTALLED_PATH: '#' + MLC_LLVM_USE_INSTALLED_DIR: yes for-intel-mlperf-inference-v3.1-bert: adr: conda-package: