From d4edd1dccee66ba64fee32290278aeeb507121a8 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Tue, 7 Jan 2025 12:30:11 -0800 Subject: [PATCH 1/3] [Exegesis] Add `--dry-run-measurement` This flag will make llvm-exegesis run everything except the actual snippet execution. --- llvm/docs/CommandGuide/llvm-exegesis.rst | 5 ++++ .../llvm-exegesis/dry-run-measurement.test | 11 +++++++++ .../llvm-exegesis/lib/BenchmarkRunner.cpp | 24 +++++++++++++++---- 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 llvm/test/tools/llvm-exegesis/dry-run-measurement.test diff --git a/llvm/docs/CommandGuide/llvm-exegesis.rst b/llvm/docs/CommandGuide/llvm-exegesis.rst index 8266d891a5e6b..cd0bce9e2dbcc 100644 --- a/llvm/docs/CommandGuide/llvm-exegesis.rst +++ b/llvm/docs/CommandGuide/llvm-exegesis.rst @@ -449,6 +449,11 @@ OPTIONS crash when hardware performance counters are unavailable and for debugging :program:`llvm-exegesis` itself. +.. option:: --dry-run-measurement + If set, llvm-exegesis runs everything except the actual snippet execution. + This is useful if we want to test some part of the code without actually + running on native platforms. + .. option:: --execution-mode=[inprocess,subprocess] This option specifies what execution mode to use. The `inprocess` execution diff --git a/llvm/test/tools/llvm-exegesis/dry-run-measurement.test b/llvm/test/tools/llvm-exegesis/dry-run-measurement.test new file mode 100644 index 0000000000000..82857e7998b5e --- /dev/null +++ b/llvm/test/tools/llvm-exegesis/dry-run-measurement.test @@ -0,0 +1,11 @@ +# RUN: llvm-exegesis --mtriple=riscv64 --mcpu=sifive-p470 --mode=latency --opcode-name=ADD --use-dummy-perf-counters --dry-run-measurement | FileCheck %s +# REQUIRES: riscv-registered-target + +# This test makes sure that llvm-exegesis doesn't execute "cross-compiled" snippets in the presence of +# --dry-run-measurement. RISC-V was chosen simply because most of the time we run tests on X86 machines. + +# Should not contain misleading results. +# CHECK: measurements: [] + +# Should not contain error messages like "snippet crashed while running: Segmentation fault". +# CHECK: error: '' diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index a7771b99e97b1..9b978c558c1fe 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -53,6 +53,12 @@ namespace llvm { namespace exegesis { +static cl::opt + DryRunMeasurement("dry-run-measurement", + cl::desc("Run every steps in the measurement phase " + "except executing the snippet."), + cl::init(false), cl::Hidden); + BenchmarkRunner::BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, ExecutionModeE ExecutionMode, @@ -140,13 +146,21 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor { Scratch->clear(); { auto PS = ET.withSavedState(); + // We can't directly capture DryRunMeasurement in the lambda below. + bool DryRun = DryRunMeasurement; CrashRecoveryContext CRC; CrashRecoveryContext::Enable(); - const bool Crashed = !CRC.RunSafely([this, Counter, ScratchPtr]() { - Counter->start(); - this->Function(ScratchPtr); - Counter->stop(); - }); + const bool Crashed = + !CRC.RunSafely([this, Counter, ScratchPtr, DryRun]() { + if (DryRun) { + Counter->start(); + Counter->stop(); + } else { + Counter->start(); + this->Function(ScratchPtr); + Counter->stop(); + } + }); CrashRecoveryContext::Disable(); PS.reset(); if (Crashed) { From a5403854cfa2d49d1ed69413bfa6194a7ab2c012 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Tue, 7 Jan 2025 16:41:07 -0800 Subject: [PATCH 2/3] Use benchmark phase instead of command line flag to select dry-run-measurement --- llvm/docs/CommandGuide/llvm-exegesis.rst | 6 +-- .../llvm-exegesis/dry-run-measurement.test | 2 +- .../tools/llvm-exegesis/lib/BenchmarkResult.h | 1 + .../llvm-exegesis/lib/BenchmarkRunner.cpp | 51 ++++++++++--------- llvm/tools/llvm-exegesis/llvm-exegesis.cpp | 5 +- 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/llvm/docs/CommandGuide/llvm-exegesis.rst b/llvm/docs/CommandGuide/llvm-exegesis.rst index cd0bce9e2dbcc..d357c2ceea418 100644 --- a/llvm/docs/CommandGuide/llvm-exegesis.rst +++ b/llvm/docs/CommandGuide/llvm-exegesis.rst @@ -301,6 +301,7 @@ OPTIONS * ``prepare-and-assemble-snippet``: Same as ``prepare-snippet``, but also dumps an excerpt of the sequence (hex encoded). * ``assemble-measured-code``: Same as ``prepare-and-assemble-snippet``. but also creates the full sequence that can be dumped to a file using ``--dump-object-to-disk``. * ``measure``: Same as ``assemble-measured-code``, but also runs the measurement. + * ``dry-run-measurement``: Same as measure, but does not actually execute the snippet. .. option:: --x86-lbr-sample-period= @@ -449,11 +450,6 @@ OPTIONS crash when hardware performance counters are unavailable and for debugging :program:`llvm-exegesis` itself. -.. option:: --dry-run-measurement - If set, llvm-exegesis runs everything except the actual snippet execution. - This is useful if we want to test some part of the code without actually - running on native platforms. - .. option:: --execution-mode=[inprocess,subprocess] This option specifies what execution mode to use. The `inprocess` execution diff --git a/llvm/test/tools/llvm-exegesis/dry-run-measurement.test b/llvm/test/tools/llvm-exegesis/dry-run-measurement.test index 82857e7998b5e..e4449d7df3d82 100644 --- a/llvm/test/tools/llvm-exegesis/dry-run-measurement.test +++ b/llvm/test/tools/llvm-exegesis/dry-run-measurement.test @@ -1,4 +1,4 @@ -# RUN: llvm-exegesis --mtriple=riscv64 --mcpu=sifive-p470 --mode=latency --opcode-name=ADD --use-dummy-perf-counters --dry-run-measurement | FileCheck %s +# RUN: llvm-exegesis --mtriple=riscv64 --mcpu=sifive-p470 --mode=latency --opcode-name=ADD --use-dummy-perf-counters --benchmark-phase=dry-run-measurement | FileCheck %s # REQUIRES: riscv-registered-target # This test makes sure that llvm-exegesis doesn't execute "cross-compiled" snippets in the presence of diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h index 3c09a8380146e..5480d85616878 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h @@ -38,6 +38,7 @@ enum class BenchmarkPhaseSelectorE { PrepareAndAssembleSnippet, AssembleMeasuredCode, Measure, + DryRunMeasure, }; enum class BenchmarkFilter { All, RegOnly, WithMem }; diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 9b978c558c1fe..cc46f7feb6cf7 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -53,12 +53,6 @@ namespace llvm { namespace exegesis { -static cl::opt - DryRunMeasurement("dry-run-measurement", - cl::desc("Run every steps in the measurement phase " - "except executing the snippet."), - cl::init(false), cl::Hidden); - BenchmarkRunner::BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, ExecutionModeE ExecutionMode, @@ -105,7 +99,7 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor { static Expected> create(const LLVMState &State, object::OwningBinary Obj, BenchmarkRunner::ScratchSpace *Scratch, - std::optional BenchmarkProcessCPU) { + std::optional BenchmarkProcessCPU, bool DryRun) { Expected EF = ExecutableFunction::create(State.createTargetMachine(), std::move(Obj)); @@ -113,14 +107,17 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor { return EF.takeError(); return std::unique_ptr( - new InProcessFunctionExecutorImpl(State, std::move(*EF), Scratch)); + new InProcessFunctionExecutorImpl(State, std::move(*EF), Scratch, + DryRun)); } private: InProcessFunctionExecutorImpl(const LLVMState &State, ExecutableFunction Function, - BenchmarkRunner::ScratchSpace *Scratch) - : State(State), Function(std::move(Function)), Scratch(Scratch) {} + BenchmarkRunner::ScratchSpace *Scratch, + bool DryRun) + : State(State), Function(std::move(Function)), Scratch(Scratch), + DryRun(DryRun) {} static void accumulateCounterValues(const SmallVector &NewValues, SmallVector *Result) { @@ -146,21 +143,18 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor { Scratch->clear(); { auto PS = ET.withSavedState(); - // We can't directly capture DryRunMeasurement in the lambda below. - bool DryRun = DryRunMeasurement; CrashRecoveryContext CRC; CrashRecoveryContext::Enable(); - const bool Crashed = - !CRC.RunSafely([this, Counter, ScratchPtr, DryRun]() { - if (DryRun) { - Counter->start(); - Counter->stop(); - } else { - Counter->start(); - this->Function(ScratchPtr); - Counter->stop(); - } - }); + const bool Crashed = !CRC.RunSafely([this, Counter, ScratchPtr]() { + if (DryRun) { + Counter->start(); + Counter->stop(); + } else { + Counter->start(); + this->Function(ScratchPtr); + Counter->stop(); + } + }); CrashRecoveryContext::Disable(); PS.reset(); if (Crashed) { @@ -191,6 +185,7 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor { const LLVMState &State; const ExecutableFunction Function; BenchmarkRunner::ScratchSpace *const Scratch; + bool DryRun = false; }; #ifdef __linux__ @@ -678,6 +673,9 @@ Expected> BenchmarkRunner::createFunctionExecutor( object::OwningBinary ObjectFile, const BenchmarkKey &Key, std::optional BenchmarkProcessCPU) const { + bool DryRun = + BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::DryRunMeasure; + switch (ExecutionMode) { case ExecutionModeE::InProcess: { if (BenchmarkProcessCPU.has_value()) @@ -685,7 +683,8 @@ BenchmarkRunner::createFunctionExecutor( "support benchmark core pinning."); auto InProcessExecutorOrErr = InProcessFunctionExecutorImpl::create( - State, std::move(ObjectFile), Scratch.get(), BenchmarkProcessCPU); + State, std::move(ObjectFile), Scratch.get(), BenchmarkProcessCPU, + DryRun); if (!InProcessExecutorOrErr) return InProcessExecutorOrErr.takeError(); @@ -693,6 +692,10 @@ BenchmarkRunner::createFunctionExecutor( } case ExecutionModeE::SubProcess: { #ifdef __linux__ + if (DryRun) + return make_error("The subprocess execution mode cannot " + "dry-run measurement at this moment."); + auto SubProcessExecutorOrErr = SubProcessFunctionExecutorImpl::create( State, std::move(ObjectFile), Key, BenchmarkProcessCPU); if (!SubProcessExecutorOrErr) diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp index fa37e05956be8..2ae76914ac2b4 100644 --- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp +++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp @@ -132,7 +132,10 @@ static cl::opt BenchmarkPhaseSelector( clEnumValN( BenchmarkPhaseSelectorE::Measure, "measure", "Same as prepare-measured-code, but also runs the measurement " - "(default)")), + "(default)"), + clEnumValN( + BenchmarkPhaseSelectorE::DryRunMeasure, "dry-run-measurement", + "Same as measure, but does not actually execute the snippet")), cl::init(BenchmarkPhaseSelectorE::Measure)); static cl::opt From 36f3dc1e60ce14c095a62095c73f3e92ec182393 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Tue, 7 Jan 2025 16:56:46 -0800 Subject: [PATCH 3/3] fixup! Use benchmark phase instead of command line flag to select dry-run-measurement --- llvm/tools/llvm-exegesis/lib/Target.cpp | 4 ++-- llvm/tools/llvm-exegesis/llvm-exegesis.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp index 29e58692f0e92..e2251ff978888 100644 --- a/llvm/tools/llvm-exegesis/lib/Target.cpp +++ b/llvm/tools/llvm-exegesis/lib/Target.cpp @@ -98,7 +98,7 @@ ExegesisTarget::createBenchmarkRunner( return nullptr; case Benchmark::Latency: case Benchmark::InverseThroughput: - if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && + if (BenchmarkPhaseSelector >= BenchmarkPhaseSelectorE::Measure && !PfmCounters.CycleCounter) { const char *ModeName = Mode == Benchmark::Latency ? "latency" @@ -116,7 +116,7 @@ ExegesisTarget::createBenchmarkRunner( State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode, ValidationCounters, BenchmarkRepeatCount); case Benchmark::Uops: - if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && + if (BenchmarkPhaseSelector >= BenchmarkPhaseSelectorE::Measure && !PfmCounters.UopsCounter && !PfmCounters.IssueCounters) return make_error( "can't run 'uops' mode, sched model does not define uops or issue " diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp index 2ae76914ac2b4..07bd44ee64f1f 100644 --- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp +++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp @@ -479,7 +479,7 @@ static void runBenchmarkConfigurations( } void benchmarkMain() { - if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && + if (BenchmarkPhaseSelector >= BenchmarkPhaseSelectorE::Measure && !UseDummyPerfCounters) { #ifndef HAVE_LIBPFM ExitWithError( @@ -504,7 +504,7 @@ void benchmarkMain() { // Preliminary check to ensure features needed for requested // benchmark mode are present on target CPU and/or OS. - if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure) + if (BenchmarkPhaseSelector >= BenchmarkPhaseSelectorE::Measure) ExitOnErr(State.getExegesisTarget().checkFeatureSupport()); if (ExecutionMode == BenchmarkRunner::ExecutionModeE::SubProcess &&