From 7c24df9fe07e42fef81be76ca631878efb30d716 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Thu, 9 Jul 2026 10:32:42 -0600 Subject: [PATCH 1/5] task: refactor ASV benchmarks --- .gitignore | 3 + benchmarks/README.md | 107 ++++++++++++ benchmarks/asv.conf.json | 93 ++-------- benchmarks/benchmarks/bench_dpbench.py | 106 +++++++++++ benchmarks/benchmarks/benchmark_utils.py | 44 +++++ benchmarks/benchmarks/common.py | 10 +- benchmarks/benchmarks/dpbench/__init__.py | 34 ++++ .../benchmarks/dpbench/_dpbench_runner.py | 165 ++++++++++++++++++ .../benchmarks/dpbench/workloads/__init__.py | 61 +++++++ .../dpbench/workloads/black_scholes.py | 135 ++++++++++++++ .../benchmarks/dpbench/workloads/gpairs.py | 156 +++++++++++++++++ .../benchmarks/dpbench/workloads/l2_norm.py | 78 +++++++++ .../dpbench/workloads/pairwise_distance.py | 84 +++++++++ .../benchmarks/dpbench/workloads/rambo.py | 89 ++++++++++ pyproject.toml | 4 + 15 files changed, 1089 insertions(+), 80 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/benchmarks/bench_dpbench.py create mode 100644 benchmarks/benchmarks/benchmark_utils.py create mode 100644 benchmarks/benchmarks/dpbench/__init__.py create mode 100644 benchmarks/benchmarks/dpbench/_dpbench_runner.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/__init__.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/black_scholes.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/gpairs.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/l2_norm.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/rambo.py diff --git a/.gitignore b/.gitignore index f66bfbb3fdd8..368a70211931 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ build_cython cython_debug dpnp.egg-info +# Airspeed Velocity (asv) benchmark environments, results and html +benchmarks/.asv/ + # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..b1a5288b3040 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,107 @@ +# dpnp benchmarks + +Benchmarking dpnp using Airspeed Velocity. +Read more about ASV [here](https://asv.readthedocs.io/en/stable/index.html). + +## Usage + +Unlike a pure-Python project, dpnp is a SYCL/DPC++ extension that requires the +Intel oneAPI compiler and a lengthy build, so ASV does not build dpnp itself: +`build_command` in `asv.conf.json` is empty and the benchmarks are run against +an **existing environment** that already has dpnp installed. + +Create an environment +[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html) +and install the benchmarking tooling into it. Either install the `benchmark` +extra from the repo: + +```bash +pip install ".[benchmark]" +``` + +or install `asv` directly: + +```bash +conda install -c conda-forge asv +``` + +Then activate the environment and run the benchmarks against it. The simplest +way is to point ASV at the currently active environment with `--python=same`: + +```bash +conda activate dpnp_env +asv run --python=same --quick HEAD^! +``` + +Alternatively, point ASV explicitly at an environment's python binary: + +```bash +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python +``` + +Compare two commits or check for regressions: + +```bash +asv continuous --python=same HEAD~1 HEAD +``` + +For `level_zero` devices, you might see `USM Allocation` errors unless you use +the `asv run` command with `--launch-method spawn`. + +By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR` +environment variable to target a specific device, e.g.: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ + --launch-method spawn \ + --python=same +``` + +## Benchmarks + +### `bench_dpbench.py` -- dpBench workloads + +`bench_dpbench.py` runs a set of dpnp workloads vendored from +[dpBench](https://github.com/IntelPython/dpbench). The kernels, their data +initialization, and the data-size presets are copied from dpBench and live in +`benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark +class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the +dpBench data-size preset (`S`, `M16Gb`, `M`, `L`). + +Currently vendored workloads: + +| Workload | Domain | +| ------------------- | ------------------ | +| `black_scholes` | Finance | +| `l2_norm` | Distance Compute | +| `pairwise_distance` | Distance Compute | +| `rambo` | Particle Physics | +| `gpairs` | Astrophysics | + +Host input data is generated and copied to the device exactly the way dpBench +does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call +blocks until the device work has finished. The `time_*` methods invoke the +workload once and let ASV wall-clock-time it (handling repeats, samples and +statistics natively) -- the same end-to-end quantity dpBench itself measures, +and the same plain `time_*` style used by the mkl_fft ASV benchmarks. By +default only the small `S` preset is exercised; edit `ASV_PRESETS` in a workload +module to benchmark larger problem sizes (which may require several GiB of +device memory). + +### Other benchmark modules + +The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`, +`bench_random.py`) are plain ASV benchmarks comparing dpnp against NumPy. + +## Writing new benchmarks + +Read ASV's guidelines for writing benchmarks +[here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). + +To add another dpBench workload, copy its `_dpnp.py` kernel and +`_initialize.py` initializer into a new module under +`benchmarks/dpbench/workloads`, translate its `bench_info` TOML presets into the +module's `PRESETS`/`ASV_PRESETS` and argument-metadata constants (see the +existing workloads for the exact shape), and add the module to `WORKLOADS` in +`benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a +benchmark class for it automatically. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 3d0e7f88d55f..6741baf28355 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -1,89 +1,26 @@ { - // The version of the config file format. Do not change, unless - // you know what you are doing. "version": 1, - - // The name of the project being benchmarked "project": "dpnp", - - // The project's homepage - "project_url": "", - - // The URL or local path of the source code repository for the - // project being benchmarked + "project_url": "https://github.com/IntelPython/dpnp", "repo": "..", - - // List of branches to benchmark. If not provided, defaults to "master" - // (for git) or "tip" (for mercurial). + "show_commit_url": "https://github.com/IntelPython/dpnp/commit/", + "build_command": [], "branches": [ "HEAD" ], - - // The DVCS being used. If not set, it will be automatically - // determined from "repo" by looking at the protocol in the URL - // (if remote), or by looking for special directories, such as - // ".git" (if local). "dvcs": "git", - - // The tool to use to create environments. May be "conda", - // "virtualenv" or other value depending on the plugins in use. - // If missing or the empty string, the tool will be automatically - // determined by looking for tools on the PATH environment - // variable. - "environment_type": "virtualenv", - - // the base URL to show a commit for the project. - "show_commit_url": "", - - // The Pythons you'd like to test against. If not provided, defaults - // to the current version of Python used to run `asv`. - "pythons": [ - "3.7" + "environment_type": "conda", + "conda_channels": [ + "https://software.repos.intel.com/python/conda/", + "conda-forge" ], - - // The matrix of dependencies to test. Each key is the name of a - // package (in PyPI) and the values are version numbers. An empty - // list indicates to just test against the default (latest) - // version. - "matrix": { - "Cython": [], - }, - - // The directory (relative to the current directory) that benchmarks are - // stored in. If not provided, defaults to "benchmarks" "benchmark_dir": "benchmarks", - - // The directory (relative to the current directory) to cache the Python - // environments in. If not provided, defaults to "env" - "env_dir": "env", - - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". - "results_dir": "results", - - // The directory (relative to the current directory) that the html tree - // should be written to. If not provided, defaults to "html". - "html_dir": "html", - - // The number of characters to retain in the commit hashes. - // "hash_length": 8, - - // `asv` will cache wheels of the recent builds in each - // environment, making them faster to install next time. This is - // number of builds to keep, per environment. - "build_cache_size": 8, - - // The commits after which the regression search in `asv publish` - // should start looking for regressions. Dictionary whose keys are - // regexps matching to benchmark names, and values corresponding to - // the commit (exclusive) after which to start looking for - // regressions. The default is to start from the first commit - // with results. If the commit is `null`, regression detection is - // skipped for the matching benchmark. - // - // "regressions_first_commits": { - // "some_benchmark": "352cdf", // Consider regressions only after this - // commit - // "another_benchmark": null, // Skip regression detection altogether - // } + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "build_cache_size": 2, + "default_benchmark_timeout": 500, + "regressions_thresholds": { + ".*": 0.2 + } } diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py new file mode 100644 index 000000000000..0b6ecc6544a3 --- /dev/null +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -0,0 +1,106 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""ASV benchmarks for dpnp workloads vendored from dpBench. + +The workloads (kernels + data initialization) and their data-size presets are +copied from dpBench (https://github.com/IntelPython/dpbench); see +``benchmarks/benchmarks/dpbench``. + +Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its output, +so a single call blocks until the device work has finished. The ``time_*`` +methods below simply invoke the workload once and let ASV wall-clock-time it +(handling repeats, samples and statistics natively) -- the same end-to-end +quantity dpBench itself measures, and the same plain ``time_*`` style used by +the mkl_fft ASV benchmarks. + +A separate benchmark class is generated for each workload -- e.g. +``BlackScholes.time_black_scholes`` -- and parametrized by the data-size preset. +""" + +import dpctl + +from . import benchmark_utils as bench_utils +from .dpbench import _dpbench_runner as runner +from .dpbench.workloads import WORKLOADS + +# Default-device queue, used only to query device capabilities (e.g. fp64 +# support) so unsupported-precision workloads can be skipped. This is the +# device dpnp allocates on by default. +DEVICE_QUEUE = dpctl.SyclQueue() + + +def _camel_case(name): + """``black_scholes`` -> ``BlackScholes``, ``l2_norm`` -> ``L2Norm``.""" + return "".join(part.capitalize() for part in name.split("_")) + + +def _make_benchmark_class(workload): + """Build an ASV benchmark class for a single dpBench workload.""" + + class WorkloadBenchmark: + # The per-benchmark timeout is governed by ``default_benchmark_timeout`` + # in ``asv.conf.json``; larger presets on a busy device can take a + # while. + + params = list(workload.ASV_PRESETS) + param_names = ["preset"] + + def setup(self, preset): + # Skip on devices that do not support the workload's precision + # (e.g. no fp64), mirroring the dpctl ASV benchmarks. + float_dtype = runner.build_types_dict(workload.PRECISION)["float"] + bench_utils.skip_unsupported_dtype(DEVICE_QUEUE, float_dtype) + + self._runner = runner.WorkloadRunner(workload, preset) + self._runner.setup() + + def time_workload(self, preset): + self._runner.run() + + # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``. + WorkloadBenchmark.__name__ = _camel_case(workload.NAME) + WorkloadBenchmark.__qualname__ = WorkloadBenchmark.__name__ + + time_method = WorkloadBenchmark.time_workload + time_method.__name__ = f"time_{workload.NAME}" + setattr(WorkloadBenchmark, time_method.__name__, time_method) + del WorkloadBenchmark.time_workload + + return WorkloadBenchmark + + +def _generate_benchmark_classes(): + """Create and register a benchmark class for every vendored workload.""" + for workload in WORKLOADS: + cls = _make_benchmark_class(workload) + # Register the class at module scope so ASV can discover it. + globals()[cls.__name__] = cls + + +_generate_benchmark_classes() diff --git a/benchmarks/benchmarks/benchmark_utils.py b/benchmarks/benchmarks/benchmark_utils.py new file mode 100644 index 000000000000..6cfeafbab990 --- /dev/null +++ b/benchmarks/benchmarks/benchmark_utils.py @@ -0,0 +1,44 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +from asv_runner.benchmarks.mark import SkipNotImplemented + +import dpnp + + +def skip_unsupported_dtype(q, dtype): + """Skip the benchmark if the device does not support the given dtype.""" + dtype = dpnp.dtype(dtype) + device = q.sycl_device + if ( + dtype in (dpnp.float64, dpnp.complex128) and not device.has_aspect_fp64 + ) or (dtype == dpnp.float16 and not device.has_aspect_fp16): + raise SkipNotImplemented( + f"Skipping benchmark for {dtype.name} on this device" + + " as it is not supported." + ) diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py index ce0956cec5d6..4303ee8fb312 100644 --- a/benchmarks/benchmarks/common.py +++ b/benchmarks/benchmarks/common.py @@ -52,10 +52,16 @@ "int64", "float64", "complex64", - "longfloat", + # numpy.longfloat is an alias of numpy.longdouble that was removed in + # NumPy 2.0; use numpy.longdouble, which exists on both 1.x and 2.x. + "longdouble", "complex128", ] -if "complex256" in numpy.typeDict: +# numpy.typeDict was removed in NumPy 2.0 in favor of numpy.sctypeDict. +_numpy_type_dict = getattr(numpy, "typeDict", None) +if _numpy_type_dict is None: + _numpy_type_dict = numpy.sctypeDict +if "complex256" in _numpy_type_dict: TYPES1.append("complex256") diff --git a/benchmarks/benchmarks/dpbench/__init__.py b/benchmarks/benchmarks/dpbench/__init__.py new file mode 100644 index 000000000000..a5c701166a4b --- /dev/null +++ b/benchmarks/benchmarks/dpbench/__init__.py @@ -0,0 +1,34 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpBench-derived ASV benchmarks for dpnp. + +This sub-package vendors a handful of dpnp workloads from dpBench +(https://github.com/IntelPython/dpbench) and exposes them as Airspeed Velocity +benchmarks. See ``benchmarks/README.md`` for details. +""" diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py new file mode 100644 index 000000000000..3f3ce641b4e8 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -0,0 +1,165 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Minimal re-implementation of dpBench's benchmark execution model for ASV. + +dpBench (https://github.com/IntelPython/dpbench) drives its benchmarks through +a fairly heavy runner that spawns a sub-process per framework, resolves TOML +configuration, validates results against a reference and persists timings to a +database. None of that machinery is importable in a lightweight ASV +environment (it pulls in ``numba_dpex``, ``sqlalchemy``, ``alembic`` and more), +so this module re-implements just the parts that matter for benchmarking: + +* data initialization -- the host (NumPy) input data is produced exactly the + way dpBench produces it, using each workload's ``initialize`` function and a + precision-driven ``types_dict`` (see ``dpbench.infrastructure.benchmark``); +* host-to-device transfer -- array arguments are copied to the device with the + same ``dpnp.asarray`` logic dpBench's ``DpnpFramework.copy_to_func`` uses; +* execution -- the dpnp implementation is invoked and blocks on device + completion (each vendored kernel ends with ``dpnp.synchronize_array_data``), + matching how dpBench itself times the workload. +""" + +import numpy + +import dpnp + +# Precision -> dtype mapping, copied from dpBench's +# ``dpbench/configs/precision_dtypes.toml``. +PRECISION_DTYPES = { + "int": {"single": "i4", "double": "i8"}, + "float": {"single": "f4", "double": "f8"}, +} + + +def build_types_dict(precision): + """Build the ``types_dict`` passed to a workload's ``initialize``. + + Mirrors ``Benchmark._get_types_dict`` in dpBench. + """ + return { + kind: numpy.dtype(precision_strings[precision]) + for kind, precision_strings in PRECISION_DTYPES.items() + } + + +def initialize_host_data(workload, preset): + """Produce the host (NumPy) input data for ``workload`` at ``preset``. + + Mirrors ``Benchmark.initialize_input_data`` / + ``_initialize_input_data_from_init`` in dpBench. + """ + if preset not in workload.PRESETS: + raise NotImplementedError( + f"{workload.NAME} doesn't have a {preset} preset." + ) + + # Preset parameters (scalars such as ``nopt``, ``seed``, ``nbins``, ...). + data = dict(workload.PRESETS[preset]) + + # The precision-driven types dictionary, if the workload's ``initialize`` + # consumes one. + if "types_dict" in workload.INIT_INPUT_ARGS: + data["types_dict"] = build_types_dict(workload.PRECISION) + + # Call ``initialize`` and store its outputs under the configured names. + init_kwargs = {arg: data[arg] for arg in workload.INIT_INPUT_ARGS} + initialized = workload.initialize(**init_kwargs) + + if isinstance(initialized, tuple): + for name, value in zip(workload.INIT_OUTPUT_ARGS, initialized): + data[name] = value + elif len(workload.INIT_OUTPUT_ARGS) == 1: + data[workload.INIT_OUTPUT_ARGS[0]] = initialized + else: + raise ValueError("Unsupported initialize output") + + return data + + +def _copy_to_device(ref_array): + """Copy a host array to the (default) device. + + Mirrors ``DpnpFramework.copy_to_func`` in dpBench. + """ + if ref_array.flags["C_CONTIGUOUS"]: + order = "C" + elif ref_array.flags["F_CONTIGUOUS"]: + order = "F" + else: + order = "K" + return dpnp.asarray( + ref_array, + dtype=ref_array.dtype, + order=order, + ) + + +def set_input_args(workload, host_data): + """Build the kernel keyword arguments, copying array args to the device. + + Mirrors ``_set_input_args`` in dpBench. + """ + inputs = {} + for arg in workload.INPUT_ARGS: + if arg in workload.ARRAY_ARGS: + inputs[arg] = _copy_to_device(host_data[arg]) + else: + inputs[arg] = host_data[arg] + return inputs + + +class WorkloadRunner: + """Sets up and runs a single dpBench workload for one preset. + + Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its + output, so a single :meth:`run` call blocks until the device work has + completed. ASV wall-clock-times the ``time_*`` method that calls + :meth:`run`, and thus captures the end-to-end (host dispatch + device) + execution time of the workload -- the same quantity dpBench measures. + """ + + def __init__(self, workload, preset): + self.workload = workload + self.preset = preset + + self.fn = getattr(workload, workload.NAME) + self.kwargs = None + + def setup(self): + """Initialize host data, transfer it to the device and warm up.""" + host_data = initialize_host_data(self.workload, self.preset) + inputs = set_input_args(self.workload, host_data) + self.kwargs = {arg: inputs[arg] for arg in self.workload.INPUT_ARGS} + + # Warmup (equivalent to dpBench's warmup step in ``_exec``). + self.run() + + def run(self): + """Execute the kernel once, blocking on device completion.""" + self.fn(**self.kwargs) diff --git a/benchmarks/benchmarks/dpbench/workloads/__init__.py b/benchmarks/benchmarks/dpbench/workloads/__init__.py new file mode 100644 index 000000000000..84be3337daf0 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/__init__.py @@ -0,0 +1,61 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpnp workloads vendored from dpBench. + +Each module exposes a uniform interface consumed by ``_dpbench_runner``: + +* ``NAME`` -- workload name; also the name of the kernel function; +* ``PRECISION`` -- ``"single"`` or ``"double"``; +* ``INPUT_ARGS`` / ``ARRAY_ARGS`` / ``OUTPUT_ARGS`` -- kernel argument metadata; +* ``INIT_INPUT_ARGS`` / ``INIT_OUTPUT_ARGS`` -- ``initialize`` argument metadata; +* ``PRESETS`` -- all dpBench data-size presets (S, M16Gb, M, L); +* ``ASV_PRESETS`` -- the subset of presets exercised by ASV; +* ``initialize(...)`` -- host data generator; +* ``(...)`` -- the dpnp kernel. +""" + +from . import black_scholes, gpairs, l2_norm, pairwise_distance, rambo + +# All vendored workloads, in a stable order. +WORKLOADS = [ + black_scholes, + l2_norm, + pairwise_distance, + rambo, + gpairs, +] + +__all__ = [ + "WORKLOADS", + "black_scholes", + "l2_norm", + "pairwise_distance", + "rambo", + "gpairs", +] diff --git a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py new file mode 100644 index 000000000000..67ab3466f8d8 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py @@ -0,0 +1,135 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Black-Scholes formula workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/black_scholes.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see black_scholes.toml) -------------------- + +NAME = "black_scholes" +PRECISION = "double" + +# Arguments passed to the kernel, in order. +INPUT_ARGS = [ + "nopt", + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] +# Arguments that are arrays and therefore copied to the device. +ARRAY_ARGS = ["price", "strike", "t", "call", "put"] +# Arguments that the kernel writes into. +OUTPUT_ARGS = ["call", "put"] + +# Arguments passed to ``initialize`` and the values it returns, in order. +INIT_INPUT_ARGS = ["nopt", "seed", "types_dict"] +INIT_OUTPUT_ARGS = [ + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] + +# Data-size presets, copied verbatim from dpBench. +PRESETS = { + "S": {"nopt": 524288, "seed": 777777}, + "M16Gb": {"nopt": 67108864, "seed": 777777}, + "M": {"nopt": 134217728, "seed": 777777}, + "L": {"nopt": 268435456, "seed": 777777}, +} +# Presets actually exercised by ASV. Larger presets require several GiB of +# device memory; add them here to benchmark bigger problem sizes. +ASV_PRESETS = ["S"] + + +def initialize(nopt, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype: np.dtype = types_dict["float"] + S0L = dtype.type(10.0) + S0H = dtype.type(50.0) + XL = dtype.type(10.0) + XH = dtype.type(50.0) + TL = dtype.type(1.0) + TH = dtype.type(2.0) + RISK_FREE = dtype.type(0.1) + VOLATILITY = dtype.type(0.2) + + default_rng.seed(seed) + price = default_rng.uniform(S0L, S0H, nopt).astype(dtype) + strike = default_rng.uniform(XL, XH, nopt).astype(dtype) + t = default_rng.uniform(TL, TH, nopt).astype(dtype) + rate = RISK_FREE + volatility = VOLATILITY + call = np.zeros(nopt, dtype=dtype) + put = -np.ones(nopt, dtype=dtype) + + return (price, strike, t, rate, volatility, call, put) + + +def black_scholes(nopt, price, strike, t, rate, volatility, call, put): + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = np.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = np.true_divide(1.0, np.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * np.scipy.special.erf(w1) + d2 = 0.5 + 0.5 * np.scipy.special.erf(w2) + + Se = np.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se + + np.synchronize_array_data(put) diff --git a/benchmarks/benchmarks/dpbench/workloads/gpairs.py b/benchmarks/benchmarks/dpbench/workloads/gpairs.py new file mode 100644 index 000000000000..fac8475cca94 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/gpairs.py @@ -0,0 +1,156 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""GPairs (galaxy pair counting) workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/gpairs.toml``. +""" + +import numpy + +import dpnp as np + +# --- dpBench benchmark metadata (see gpairs.toml) --------------------------- + +NAME = "gpairs" +PRECISION = "double" + +INPUT_ARGS = [ + "nopt", + "nbins", + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +ARRAY_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +OUTPUT_ARGS = ["results"] + +INIT_INPUT_ARGS = ["nopt", "seed", "nbins", "rmax", "rmin", "types_dict"] +INIT_OUTPUT_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] + +PRESETS = { + "S": {"nopt": 128, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "M16Gb": { + "nopt": 4096, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, + "M": {"nopt": 8192, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "L": { + "nopt": 524288, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, +} +ASV_PRESETS = ["S"] + + +def _generate_rbins(dtype, nbins, rmax, rmin): + rbins = numpy.logspace(numpy.log10(rmin), numpy.log10(rmax), nbins).astype( + dtype + ) + + return (rbins**2).astype(dtype) + + +def initialize(nopt, seed, nbins, rmax, rmin, types_dict): + import numpy.random as default_rng + + default_rng.seed(seed) + dtype = types_dict["float"] + x1 = numpy.random.randn(nopt).astype(dtype) + y1 = numpy.random.randn(nopt).astype(dtype) + z1 = numpy.random.randn(nopt).astype(dtype) + w1 = numpy.random.rand(nopt).astype(dtype) + w1 = w1 / numpy.sum(w1) + + x2 = numpy.random.randn(nopt).astype(dtype) + y2 = numpy.random.randn(nopt).astype(dtype) + z2 = numpy.random.randn(nopt).astype(dtype) + w2 = numpy.random.rand(nopt).astype(dtype) + w2 = w2 / numpy.sum(w2) + + rbins = _generate_rbins(dtype=dtype, rmin=rmin, rmax=rmax, nbins=nbins) + results = numpy.zeros_like(rbins).astype(dtype) + return (x1, y1, z1, w1, x2, y2, z2, w2, rbins, results) + + +def _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + np.square(x2 - x1[:, None]) + + np.square(y2 - y1[:, None]) + + np.square(z2 - z1[:, None]) + ) + return np.array( + [ + np.outer(w1, w2)[dm <= rbins[k]].sum(dtype=np.result_type(w1, w2)) + for k in range(len(rbins)) + ], + device=x1.device, + ) + + +def gpairs(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + results[:] = _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) + + np.synchronize_array_data(results) diff --git a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py new file mode 100644 index 000000000000..a059ca8d9415 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py @@ -0,0 +1,78 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""L2-norm workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/l2_norm.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see l2_norm.toml) -------------------------- + +NAME = "l2_norm" +PRECISION = "double" + +INPUT_ARGS = ["a", "d"] +ARRAY_ARGS = ["a", "d"] +OUTPUT_ARGS = ["d"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["a", "d"] + +PRESETS = { + "S": {"npoints": 32768, "dims": 3, "seed": 777777}, + "M16Gb": {"npoints": 134217728, "dims": 3, "seed": 777777}, + "M": {"npoints": 268435456, "dims": 3, "seed": 777777}, + "L": {"npoints": 536870912, "dims": 3, "seed": 777777}, +} +ASV_PRESETS = ["S"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + np.zeros(npoints).astype(dtype), + ) + + +def l2_norm(a, d): + sq = np.square(a) + sum = sq.sum(axis=1, dtype=sq.dtype) + d[:] = np.sqrt(sum) + + np.synchronize_array_data(d) diff --git a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py new file mode 100644 index 000000000000..94fd29df4143 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py @@ -0,0 +1,84 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Pairwise-distance workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see pairwise_distance.toml) ---------------- + +NAME = "pairwise_distance" +PRECISION = "double" + +INPUT_ARGS = ["X1", "X2", "D"] +ARRAY_ARGS = ["X1", "X2", "D"] +OUTPUT_ARGS = ["D"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["X1", "X2", "D"] + +PRESETS = { + "S": {"npoints": 1024, "dims": 3, "seed": 7777777}, + "M16Gb": {"npoints": 21846, "dims": 3, "seed": 7777777}, + "M": {"npoints": 32768, "dims": 3, "seed": 7777777}, + "L": {"npoints": 44032, "dims": 3, "seed": 7777777}, +} +ASV_PRESETS = ["S"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + default_rng.random((npoints, dims)).astype(dtype), + np.empty((npoints, npoints), dtype), + ) + + +def pairwise_distance(X1, X2, D): + x1 = np.sum(np.square(X1), axis=1, dtype=X1.dtype) + x2 = np.sum(np.square(X2), axis=1, dtype=X2.dtype) + np.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + np.add(D, x3, D) + np.add(D, x2, D) + np.sqrt(D, D) + + np.synchronize_array_data(D) diff --git a/benchmarks/benchmarks/dpbench/workloads/rambo.py b/benchmarks/benchmarks/dpbench/workloads/rambo.py new file mode 100644 index 000000000000..d436af9c721f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/rambo.py @@ -0,0 +1,89 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Rambo workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/rambo.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see rambo.toml) ---------------------------- + +NAME = "rambo" +PRECISION = "double" + +INPUT_ARGS = ["nevts", "nout", "C1", "F1", "Q1", "output"] +ARRAY_ARGS = ["C1", "F1", "Q1", "output"] +OUTPUT_ARGS = ["output"] + +INIT_INPUT_ARGS = ["nevts", "nout", "types_dict"] +INIT_OUTPUT_ARGS = ["C1", "F1", "Q1", "output"] + +PRESETS = { + "S": {"nevts": 32768, "nout": 4}, + "M16Gb": {"nevts": 16777216, "nout": 4}, + "M": {"nevts": 8388608, "nout": 4}, + "L": {"nevts": 16777216, "nout": 4}, +} +ASV_PRESETS = ["S"] + + +def initialize(nevts, nout, types_dict): + import numpy as np + + dtype = types_dict["float"] + + C1 = np.empty((nevts, nout), dtype=dtype) + F1 = np.empty((nevts, nout), dtype=dtype) + Q1 = np.empty((nevts, nout), dtype=dtype) + + np.random.seed(777) + for i in range(nevts): + for j in range(nout): + C1[i, j] = np.random.rand() + F1[i, j] = np.random.rand() + Q1[i, j] = np.random.rand() * np.random.rand() + + return (C1, F1, Q1, np.empty((nevts, nout, 4), dtype)) + + +def rambo(nevts, nout, C1, F1, Q1, output): + C = 2.0 * C1 - 1.0 + S = np.sqrt(1 - np.square(C)) + F = 2.0 * np.pi * F1 + Q = -np.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * np.sin(F) + output[:, :, 2] = Q * S * np.cos(F) + output[:, :, 3] = Q * C + + np.synchronize_array_data(output) diff --git a/pyproject.toml b/pyproject.toml index 773d3cb45909..f6c54ae31ee0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10,<3.15" [project.optional-dependencies] +benchmark = [ + "asv>=0.6", + "scipy" +] coverage = [ "coverage", "Cython", From f1e46a9adba055b67eb5e3d2aa866c4dec104cb7 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Wed, 12 Aug 2026 13:34:01 -0600 Subject: [PATCH 2/5] fix: benchmark logic further --- benchmarks/README.md | 77 ++++++--- benchmarks/benchmarks/bench_dpbench.py | 48 ++++-- benchmarks/benchmarks/benchmark_utils.py | 2 +- benchmarks/benchmarks/common.py | 12 +- benchmarks/benchmarks/dpbench/__init__.py | 2 +- .../benchmarks/dpbench/_dpbench_runner.py | 146 +++++++++++++++++- .../benchmarks/dpbench/workloads/__init__.py | 10 +- .../dpbench/workloads/black_scholes.py | 63 ++++++-- .../benchmarks/dpbench/workloads/gpairs.py | 37 ++++- .../benchmarks/dpbench/workloads/l2_norm.py | 32 +++- .../dpbench/workloads/pairwise_distance.py | 38 ++++- .../benchmarks/dpbench/workloads/rambo.py | 57 +++++-- pyproject.toml | 2 + 13 files changed, 426 insertions(+), 100 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index b1a5288b3040..f2972cb392ce 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -12,17 +12,22 @@ an **existing environment** that already has dpnp installed. Create an environment [following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html) -and install the benchmarking tooling into it. Either install the `benchmark` -extra from the repo: +and install the benchmarking tooling into it. + +Install the tooling directly, which leaves the already-built dpnp untouched: ```bash -pip install ".[benchmark]" +conda install -c conda-forge asv scipy ``` -or install `asv` directly: +Do **not** use `pip install ".[benchmark]"` for an environment that already has +dpnp: dpnp is a scikit-build project, so pip reinstalls the `dpnp` package +itself and triggers a full oneAPI/DPC++ rebuild of the backend just to pull in +two pure-Python dependencies. The `benchmark` extra exists for the case where +dpnp is being built from source anyway, e.g.: ```bash -conda install -c conda-forge asv +pip install --no-build-isolation --no-deps -e ".[benchmark]" ``` Then activate the environment and run the benchmarks against it. The simplest @@ -30,23 +35,28 @@ way is to point ASV at the currently active environment with `--python=same`: ```bash conda activate dpnp_env -asv run --python=same --quick HEAD^! +asv run --python=same --launch-method spawn --quick HEAD^! ``` Alternatively, point ASV explicitly at an environment's python binary: ```bash -asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ + --launch-method spawn ``` Compare two commits or check for regressions: ```bash -asv continuous --python=same HEAD~1 HEAD +asv continuous --python=same --launch-method spawn HEAD~1 HEAD ``` -For `level_zero` devices, you might see `USM Allocation` errors unless you use -the `asv run` command with `--launch-method spawn`. +**Always pass `--launch-method spawn`.** ASV defaults to a forkserver, which +`fork()`s a process that has already initialized a SYCL runtime; the SYCL +runtime is multi-threaded and not fork-safe, so benchmarks may hang until +`default_benchmark_timeout` expires (reported as `failed`) or fail with +`USM Allocation` errors on `level_zero` devices. `spawn` starts a fresh +interpreter per benchmark and avoids this entirely. By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR` environment variable to target a specific device, e.g.: @@ -66,7 +76,8 @@ ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ initialization, and the data-size presets are copied from dpBench and live in `benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the -dpBench data-size preset (`S`, `M16Gb`, `M`, `L`). +dpBench data-size preset (`S`, `M16Gb`, `M`, `L`) and by floating-point +precision (`single`, `double`). Currently vendored workloads: @@ -83,10 +94,35 @@ does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call blocks until the device work has finished. The `time_*` methods invoke the workload once and let ASV wall-clock-time it (handling repeats, samples and statistics natively) -- the same end-to-end quantity dpBench itself measures, -and the same plain `time_*` style used by the mkl_fft ASV benchmarks. By -default only the small `S` preset is exercised; edit `ASV_PRESETS` in a workload -module to benchmark larger problem sizes (which may require several GiB of -device memory). +and the same plain `time_*` style used by the mkl_fft ASV benchmarks. + +**Precision.** Both `single` and `double` are benchmarked. Devices without fp64 +support (common on iGPUs) skip the `double` parametrization via +`SkipNotImplemented` rather than failing the run, so such a device still +produces `single`-precision results. dpBench's own configs request `double` +throughout; that value is kept in each workload's `PRECISION` for reference. + +**Preset selection.** Presets are chosen per device instead of being hard-coded: +`_dpbench_runner.select_presets` keeps every preset whose estimated peak device +footprint (each workload's `peak_elements`) fits within a fraction of the +device's `global_mem_size`. A large discrete GPU therefore exercises the bigger +problem sizes automatically, while a small iGPU stays on `S`. Note that dpBench's +preset names are not ordered by size -- `M16Gb` is *smaller* than `M`. + +Prefer the largest preset your device fits when looking for regressions. The +smallest sizes are dominated by per-call dispatch overhead and are noticeably +noisier: on a CPU device the run-to-run spread of the median at `S` was measured +at 4-14%, against the 20% `regressions_thresholds` in `asv.conf.json`, whereas +the larger presets settled to a few percent. Timings at `S` are still useful for +a quick smoke test, and ASV's repeat/sample handling absorbs part of the noise. + +**Validation.** Each workload also ships the NumPy `reference` implementation +from dpBench, and every benchmark's `setup` compares the dpnp results for all +`OUTPUT_ARGS` against it (mirroring dpBench's +`infrastructure/benchmark_validation.py`, same `1e-05` relative-error +tolerance). A numerically wrong kernel therefore fails the benchmark instead of +being silently timed. Validation runs outside the timed region and does not +affect the reported numbers. ### Other benchmark modules @@ -98,10 +134,11 @@ The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`, Read ASV's guidelines for writing benchmarks [here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). -To add another dpBench workload, copy its `_dpnp.py` kernel and -`_initialize.py` initializer into a new module under -`benchmarks/dpbench/workloads`, translate its `bench_info` TOML presets into the -module's `PRESETS`/`ASV_PRESETS` and argument-metadata constants (see the -existing workloads for the exact shape), and add the module to `WORKLOADS` in +To add another dpBench workload, copy its `_dpnp.py` kernel, +`_numpy.py` reference (as `reference`) and `_initialize.py` +initializer into a new module under `benchmarks/dpbench/workloads`, translate its +`bench_info` TOML presets into the module's `PRESETS` and argument-metadata +constants, add a `peak_elements` estimate (see the existing workloads for the +exact shape), and add the module to `WORKLOADS` in `benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a benchmark class for it automatically. diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py index 0b6ecc6544a3..2e6218c9d4a6 100644 --- a/benchmarks/benchmarks/bench_dpbench.py +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -40,7 +40,19 @@ the mkl_fft ASV benchmarks. A separate benchmark class is generated for each workload -- e.g. -``BlackScholes.time_black_scholes`` -- and parametrized by the data-size preset. +``BlackScholes.time_black_scholes`` -- parametrized by the data-size preset and +the floating-point precision. The presets are chosen per device so that only +problem sizes fitting into device memory are benchmarked, and a precision the +device does not support (typically fp64 on an iGPU) is skipped rather than +failing the run. + +``setup`` also validates the dpnp results against the workload's NumPy +reference, so a numerically wrong kernel fails the benchmark instead of being +timed. Validation happens outside the timed region and therefore does not +affect the reported numbers, but it is limited to the cheapest preset: the +reference runs on the host, and at the larger presets it costs far more than +the benchmark it guards (measured at ~70 s for ``pairwise_distance`` at +``M16Gb``) while checking numerics that do not depend on the problem size. """ import dpctl @@ -49,10 +61,11 @@ from .dpbench import _dpbench_runner as runner from .dpbench.workloads import WORKLOADS -# Default-device queue, used only to query device capabilities (e.g. fp64 -# support) so unsupported-precision workloads can be skipped. This is the +# Default-device queue, used to query device capabilities (fp64 support, memory +# size) so the parameter matrix can be tailored to the device. This is the # device dpnp allocates on by default. DEVICE_QUEUE = dpctl.SyclQueue() +DEVICE = DEVICE_QUEUE.sycl_device def _camel_case(name): @@ -68,19 +81,28 @@ class WorkloadBenchmark: # in ``asv.conf.json``; larger presets on a busy device can take a # while. - params = list(workload.ASV_PRESETS) - param_names = ["preset"] + params = [ + runner.select_presets(workload, DEVICE), + list(runner.PRECISIONS), + ] + param_names = ["preset", "precision"] - def setup(self, preset): - # Skip on devices that do not support the workload's precision - # (e.g. no fp64), mirroring the dpctl ASV benchmarks. - float_dtype = runner.build_types_dict(workload.PRECISION)["float"] - bench_utils.skip_unsupported_dtype(DEVICE_QUEUE, float_dtype) + # Preset the results are validated against; see the module docstring. + _validated_preset = runner.presets_by_size(workload)[0] - self._runner = runner.WorkloadRunner(workload, preset) + def setup(self, preset, precision): + # Skip precisions the device does not support (e.g. fp64 on many + # iGPUs), mirroring the dpctl ASV benchmarks. + bench_utils.skip_unsupported_dtype( + DEVICE_QUEUE, runner.float_dtype(precision) + ) + + self._runner = runner.WorkloadRunner(workload, preset, precision) self._runner.setup() + if preset == self._validated_preset: + self._runner.validate() - def time_workload(self, preset): + def time_workload(self, preset, precision): self._runner.run() # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``. diff --git a/benchmarks/benchmarks/benchmark_utils.py b/benchmarks/benchmarks/benchmark_utils.py index 6cfeafbab990..c095c1a57ab1 100644 --- a/benchmarks/benchmarks/benchmark_utils.py +++ b/benchmarks/benchmarks/benchmark_utils.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py index 4303ee8fb312..0d708c2789d1 100644 --- a/benchmarks/benchmarks/common.py +++ b/benchmarks/benchmarks/common.py @@ -44,6 +44,9 @@ nxs, nys = 100, 100 # a set of interesting types to test +# NOTE: extended-precision types (numpy.longdouble / numpy.complex256, and the +# removed numpy.longfloat alias) are intentionally absent -- dpnp has no +# counterpart for them, so dpnp.asarray() rejects such input. TYPES1 = [ "int16", "float16", @@ -52,17 +55,8 @@ "int64", "float64", "complex64", - # numpy.longfloat is an alias of numpy.longdouble that was removed in - # NumPy 2.0; use numpy.longdouble, which exists on both 1.x and 2.x. - "longdouble", "complex128", ] -# numpy.typeDict was removed in NumPy 2.0 in favor of numpy.sctypeDict. -_numpy_type_dict = getattr(numpy, "typeDict", None) -if _numpy_type_dict is None: - _numpy_type_dict = numpy.sctypeDict -if "complex256" in _numpy_type_dict: - TYPES1.append("complex256") def memoize(func): diff --git a/benchmarks/benchmarks/dpbench/__init__.py b/benchmarks/benchmarks/dpbench/__init__.py index a5c701166a4b..def3c0ad47bb 100644 --- a/benchmarks/benchmarks/dpbench/__init__.py +++ b/benchmarks/benchmarks/dpbench/__init__.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py index 3f3ce641b4e8..ffd6b7d6e12b 100644 --- a/benchmarks/benchmarks/dpbench/_dpbench_runner.py +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -42,7 +42,10 @@ same ``dpnp.asarray`` logic dpBench's ``DpnpFramework.copy_to_func`` uses; * execution -- the dpnp implementation is invoked and blocks on device completion (each vendored kernel ends with ``dpnp.synchronize_array_data``), - matching how dpBench itself times the workload. + matching how dpBench itself times the workload; +* validation -- the dpnp results are compared against the workload's NumPy + reference implementation, mirroring + ``dpbench.infrastructure.benchmark_validation``. """ import numpy @@ -56,6 +59,19 @@ "float": {"single": "f4", "double": "f8"}, } +# Precisions ASV benchmarks each workload at. dpBench's configs request +# ``double`` throughout, but not every device supports fp64 (many iGPUs do +# not), so ``single`` is benchmarked as well and the unsupported one is skipped +# per device -- that way an fp64-less device still produces results instead of +# reporting nothing. +PRECISIONS = ["single", "double"] + +# Fraction of the device's global memory a benchmark's estimated peak +# footprint is allowed to occupy. Kept well below 1.0 because the estimates +# below only count the obvious buffers, the device is usually shared with a +# display server, and dpnp's own allocator caches freed blocks. +_MEMORY_BUDGET_FRACTION = 0.25 + def build_types_dict(precision): """Build the ``types_dict`` passed to a workload's ``initialize``. @@ -68,7 +84,52 @@ def build_types_dict(precision): } -def initialize_host_data(workload, preset): +def float_dtype(precision): + """Return the floating-point dtype used at ``precision``.""" + return build_types_dict(precision)["float"] + + +def select_presets(workload, device, precision="double"): + """Pick the dpBench presets that fit into ``device``'s global memory. + + dpBench leaves preset selection to the user; ASV needs it decided up front + because ``params`` is evaluated at import time. Every preset whose + estimated peak footprint (see each workload's ``peak_elements``) fits the + memory budget is returned, so a large discrete GPU automatically exercises + the bigger problem sizes while a small iGPU stays on ``S``. + + ``precision`` is deliberately the *widest* precision benchmarked rather + than each one separately: it keeps the preset list identical across the + precision parameter, so ASV's parameter matrix stays rectangular and + results remain comparable. + """ + itemsize = float_dtype(precision).itemsize + budget = _MEMORY_BUDGET_FRACTION * device.global_mem_size + + fitting = [ + name + for name in presets_by_size(workload) + if workload.peak_elements(workload.PRESETS[name]) * itemsize <= budget + ] + # Always benchmark something: if even the smallest preset is over budget, + # fall back to it and let the run fail loudly on allocation instead of + # silently reporting no data at all. + return fitting or presets_by_size(workload)[:1] + + +def presets_by_size(workload): + """Return the workload's preset names ordered cheapest-first. + + dpBench's preset names are not ordered by size (``M16Gb`` is smaller than + ``M``), so sort explicitly rather than relying on the declaration order. + """ + return sorted( + workload.PRESETS, + key=lambda name: workload.peak_elements(workload.PRESETS[name]), + ) + + +def initialize_host_data(workload, preset, precision): """Produce the host (NumPy) input data for ``workload`` at ``preset``. Mirrors ``Benchmark.initialize_input_data`` / @@ -85,7 +146,7 @@ def initialize_host_data(workload, preset): # The precision-driven types dictionary, if the workload's ``initialize`` # consumes one. if "types_dict" in workload.INIT_INPUT_ARGS: - data["types_dict"] = build_types_dict(workload.PRECISION) + data["types_dict"] = build_types_dict(precision) # Call ``initialize`` and store its outputs under the configured names. init_kwargs = {arg: data[arg] for arg in workload.INIT_INPUT_ARGS} @@ -134,6 +195,42 @@ def set_input_args(workload, host_data): return inputs +def relative_error(ref, val): + """Relative error between a reference and a measured array. + + Copied from ``dpbench.infrastructure.benchmark_validation``. + """ + ref_norm = numpy.linalg.norm(ref) + if ref_norm == 0: + val_norm = numpy.linalg.norm(val) + if val_norm == 0: + return 0.0 + ref_norm = val_norm + + return numpy.linalg.norm(ref - val) / ref_norm + + +def validate(expected, actual, rel_error=1e-05): + """Check that ``actual`` matches ``expected`` closely enough. + + Mirrors ``dpbench.infrastructure.benchmark_validation.validate``: a + mismatch is tolerated only while the relative error stays below + ``rel_error``. Raises :exc:`ValueError` naming the offending argument + instead of returning a bool, so a wrong result fails the benchmark rather + than being silently timed. + """ + for name, ref in expected.items(): + val = actual[name] + if numpy.allclose(ref, val): + continue + error = relative_error(ref, val) + if error >= rel_error: + raise ValueError( + f"Validation failed for {name!r}: relative error {error:.3e} " + f"exceeds the {rel_error:.0e} tolerance." + ) + + class WorkloadRunner: """Sets up and runs a single dpBench workload for one preset. @@ -144,16 +241,22 @@ class WorkloadRunner: execution time of the workload -- the same quantity dpBench measures. """ - def __init__(self, workload, preset): + def __init__(self, workload, preset, precision="double"): self.workload = workload self.preset = preset + self.precision = precision self.fn = getattr(workload, workload.NAME) self.kwargs = None def setup(self): """Initialize host data, transfer it to the device and warm up.""" - host_data = initialize_host_data(self.workload, self.preset) + # The host data is deliberately not retained: once the array arguments + # have been copied to the device it would just pin a second, host-side + # copy of the whole problem (several GiB at the larger presets). + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) inputs = set_input_args(self.workload, host_data) self.kwargs = {arg: inputs[arg] for arg in self.workload.INPUT_ARGS} @@ -163,3 +266,34 @@ def setup(self): def run(self): """Execute the kernel once, blocking on device completion.""" self.fn(**self.kwargs) + + def validate(self): + """Compare the dpnp results against the NumPy reference. + + Runs the workload's ``reference`` implementation on a fresh copy of the + same host data and compares every ``OUTPUT_ARGS`` entry, mirroring + dpBench's post-run validation step. Called from the benchmark's + ``setup``, so a numerically wrong kernel fails the benchmark instead of + being timed. + """ + expected = { + arg: value + for arg, value in self._reference_outputs().items() + if arg in self.workload.OUTPUT_ARGS + } + actual = { + arg: dpnp.asnumpy(self.kwargs[arg]) + for arg in self.workload.OUTPUT_ARGS + } + validate(expected, actual) + + def _reference_outputs(self): + """Run the NumPy reference on freshly initialized host data.""" + # A fresh initialization is required: the kernel writes into its output + # arrays, so ``self._host_data`` no longer holds their initial values. + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) + kwargs = {arg: host_data[arg] for arg in self.workload.INPUT_ARGS} + self.workload.reference(**kwargs) + return kwargs diff --git a/benchmarks/benchmarks/dpbench/workloads/__init__.py b/benchmarks/benchmarks/dpbench/workloads/__init__.py index 84be3337daf0..f6823188e294 100644 --- a/benchmarks/benchmarks/dpbench/workloads/__init__.py +++ b/benchmarks/benchmarks/dpbench/workloads/__init__.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,13 +31,15 @@ Each module exposes a uniform interface consumed by ``_dpbench_runner``: * ``NAME`` -- workload name; also the name of the kernel function; -* ``PRECISION`` -- ``"single"`` or ``"double"``; +* ``PRECISION`` -- the precision dpBench's config requests for this workload; * ``INPUT_ARGS`` / ``ARRAY_ARGS`` / ``OUTPUT_ARGS`` -- kernel argument metadata; * ``INIT_INPUT_ARGS`` / ``INIT_OUTPUT_ARGS`` -- ``initialize`` argument metadata; * ``PRESETS`` -- all dpBench data-size presets (S, M16Gb, M, L); -* ``ASV_PRESETS`` -- the subset of presets exercised by ASV; +* ``peak_elements(params)`` -- estimated peak device element count for a preset, + used to pick the presets that fit into the device's memory; * ``initialize(...)`` -- host data generator; -* ``(...)`` -- the dpnp kernel. +* ``(...)`` -- the dpnp kernel; +* ``reference(...)`` -- the NumPy kernel the dpnp results are validated against. """ from . import black_scholes, gpairs, l2_norm, pairwise_distance, rambo diff --git a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py index 67ab3466f8d8..dd4aa6d6de2f 100644 --- a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py +++ b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Black-Scholes formula workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/black_scholes.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/black_scholes.toml``. """ import dpnp as np @@ -38,6 +38,9 @@ # --- dpBench benchmark metadata (see black_scholes.toml) -------------------- NAME = "black_scholes" +# Precision requested by the dpBench config. ASV benchmarks every precision in +# ``_dpbench_runner.PRECISIONS`` that the device supports, so this is only the +# documented dpBench default. PRECISION = "double" # Arguments passed to the kernel, in order. @@ -75,16 +78,23 @@ "M": {"nopt": 134217728, "seed": 777777}, "L": {"nopt": 268435456, "seed": 777777}, } -# Presets actually exercised by ASV. Larger presets require several GiB of -# device memory; add them here to benchmark bigger problem sizes. -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + 5 input/output arrays of ``nopt`` elements, plus the ~8 temporaries the + kernel below materializes (``a``, ``b``, ``z``, ``c``, ``y``, ``w1``, + ``w2``, ``Se``, ...). + """ + return 13 * params["nopt"] def initialize(nopt, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng - dtype: np.dtype = types_dict["float"] + dtype: numpy.dtype = types_dict["float"] S0L = dtype.type(10.0) S0H = dtype.type(50.0) XL = dtype.type(10.0) @@ -100,8 +110,8 @@ def initialize(nopt, seed, types_dict): t = default_rng.uniform(TL, TH, nopt).astype(dtype) rate = RISK_FREE volatility = VOLATILITY - call = np.zeros(nopt, dtype=dtype) - put = -np.ones(nopt, dtype=dtype) + call = numpy.zeros(nopt, dtype=dtype) + put = -numpy.ones(nopt, dtype=dtype) return (price, strike, t, rate, volatility, call, put) @@ -133,3 +143,34 @@ def black_scholes(nopt, price, strike, t, rate, volatility, call, put): put[:] = call - P + Se np.synchronize_array_data(put) + + +def reference(nopt, price, strike, t, rate, volatility, call, put): + """NumPy reference, copied from dpBench's ``black_scholes_numpy.py``.""" + import numpy + from scipy.special import erf + + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = numpy.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = numpy.true_divide(1.0, numpy.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * erf(w1) + d2 = 0.5 + 0.5 * erf(w2) + + Se = numpy.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se diff --git a/benchmarks/benchmarks/dpbench/workloads/gpairs.py b/benchmarks/benchmarks/dpbench/workloads/gpairs.py index fac8475cca94..27509f85739f 100644 --- a/benchmarks/benchmarks/dpbench/workloads/gpairs.py +++ b/benchmarks/benchmarks/dpbench/workloads/gpairs.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """GPairs (galaxy pair counting) workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/gpairs.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/gpairs.toml``. """ import numpy @@ -40,6 +40,7 @@ # --- dpBench benchmark metadata (see gpairs.toml) --------------------------- NAME = "gpairs" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = [ @@ -102,7 +103,17 @@ "rmin": 0.1, }, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(nopt, nopt)`` distance matrix ``dm``; the kernel also + materializes a same-shaped ``outer(w1, w2)`` and a boolean mask of the same + extent once per bin, hence the factor of 3. + """ + nopt = params["nopt"] + return 3 * nopt * nopt + 8 * nopt def _generate_rbins(dtype, nbins, rmax, rmin): @@ -154,3 +165,19 @@ def gpairs(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): results[:] = _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) np.synchronize_array_data(results) + + +def _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + numpy.square(x2 - x1[:, None]) + + numpy.square(y2 - y1[:, None]) + + numpy.square(z2 - z1[:, None]) + ) + return numpy.array( + [numpy.outer(w1, w2)[dm <= rbins[k]].sum() for k in range(len(rbins))] + ) + + +def reference(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + """NumPy reference, copied from dpBench's ``gpairs_numpy.py``.""" + results[:] = _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) diff --git a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py index a059ca8d9415..077c87e55d6c 100644 --- a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py +++ b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """L2-norm workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/l2_norm.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/l2_norm.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see l2_norm.toml) -------------------------- NAME = "l2_norm" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["a", "d"] @@ -53,11 +54,19 @@ "M": {"npoints": 268435456, "dims": 3, "seed": 777777}, "L": {"npoints": 536870912, "dims": 3, "seed": 777777}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(npoints, dims)`` input plus the same-shaped ``sq`` temporary, and + two ``npoints``-sized vectors (``d`` and the ``sum`` reduction). + """ + return 2 * params["npoints"] * params["dims"] + 2 * params["npoints"] def initialize(npoints, dims, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng dtype = types_dict["float"] @@ -66,7 +75,7 @@ def initialize(npoints, dims, seed, types_dict): return ( default_rng.random((npoints, dims)).astype(dtype), - np.zeros(npoints).astype(dtype), + numpy.zeros(npoints).astype(dtype), ) @@ -76,3 +85,12 @@ def l2_norm(a, d): d[:] = np.sqrt(sum) np.synchronize_array_data(d) + + +def reference(a, d): + """NumPy reference, copied from dpBench's ``l2_norm_numpy.py``.""" + import numpy + + sq = numpy.square(a) + sum = sq.sum(axis=1) + d[:] = numpy.sqrt(sum) diff --git a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py index 94fd29df4143..4067764bd0ed 100644 --- a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py +++ b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Pairwise-distance workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see pairwise_distance.toml) ---------------- NAME = "pairwise_distance" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["X1", "X2", "D"] @@ -53,11 +54,20 @@ "M": {"npoints": 32768, "dims": 3, "seed": 7777777}, "L": {"npoints": 44032, "dims": 3, "seed": 7777777}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(npoints, npoints)`` distance matrix ``D``; the two + ``(npoints, dims)`` inputs are negligible in comparison but counted anyway. + """ + npoints = params["npoints"] + return npoints * npoints + 2 * npoints * params["dims"] def initialize(npoints, dims, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng dtype = types_dict["float"] @@ -67,7 +77,7 @@ def initialize(npoints, dims, seed, types_dict): return ( default_rng.random((npoints, dims)).astype(dtype), default_rng.random((npoints, dims)).astype(dtype), - np.empty((npoints, npoints), dtype), + numpy.empty((npoints, npoints), dtype), ) @@ -82,3 +92,17 @@ def pairwise_distance(X1, X2, D): np.sqrt(D, D) np.synchronize_array_data(D) + + +def reference(X1, X2, D): + """NumPy reference, copied from dpBench's ``pairwise_distance_numpy.py``.""" + import numpy + + x1 = numpy.sum(numpy.square(X1), axis=1) + x2 = numpy.sum(numpy.square(X2), axis=1) + numpy.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + numpy.add(D, x3, D) + numpy.add(D, x2, D) + numpy.sqrt(D, D) diff --git a/benchmarks/benchmarks/dpbench/workloads/rambo.py b/benchmarks/benchmarks/dpbench/workloads/rambo.py index d436af9c721f..87572c51bfad 100644 --- a/benchmarks/benchmarks/dpbench/workloads/rambo.py +++ b/benchmarks/benchmarks/dpbench/workloads/rambo.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Rambo workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/rambo.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied from dpBench (https://github.com/IntelPython/dpbench), and the metadata +below mirrors ``dpbench/configs/bench_info/rambo.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see rambo.toml) ---------------------------- NAME = "rambo" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["nevts", "nout", "C1", "F1", "Q1", "output"] @@ -53,26 +54,35 @@ "M": {"nevts": 8388608, "nout": 4}, "L": {"nevts": 16777216, "nout": 4}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(nevts, nout, 4)`` output, the three ``(nevts, nout)`` inputs and the + ~6 same-shaped temporaries the kernel materializes (``C``, ``S``, ``F``, + ``Q``, and the ``sin``/``cos`` results). + """ + return 13 * params["nevts"] * params["nout"] def initialize(nevts, nout, types_dict): - import numpy as np + import numpy dtype = types_dict["float"] - C1 = np.empty((nevts, nout), dtype=dtype) - F1 = np.empty((nevts, nout), dtype=dtype) - Q1 = np.empty((nevts, nout), dtype=dtype) + # dpBench draws these element-by-element in a Python loop; drawing the + # whole block at once consumes the same RNG stream in the same order (so + # the data is bit-identical) but is orders of magnitude faster, which + # matters because ASV re-runs ``setup`` for every benchmark round. + numpy.random.seed(777) + draws = numpy.random.rand(nevts, nout, 4) - np.random.seed(777) - for i in range(nevts): - for j in range(nout): - C1[i, j] = np.random.rand() - F1[i, j] = np.random.rand() - Q1[i, j] = np.random.rand() * np.random.rand() + C1 = draws[..., 0].astype(dtype) + F1 = draws[..., 1].astype(dtype) + Q1 = (draws[..., 2] * draws[..., 3]).astype(dtype) - return (C1, F1, Q1, np.empty((nevts, nout, 4), dtype)) + return (C1, F1, Q1, numpy.empty((nevts, nout, 4), dtype)) def rambo(nevts, nout, C1, F1, Q1, output): @@ -87,3 +97,18 @@ def rambo(nevts, nout, C1, F1, Q1, output): output[:, :, 3] = Q * C np.synchronize_array_data(output) + + +def reference(nevts, nout, C1, F1, Q1, output): + """NumPy reference, copied from dpBench's ``rambo_numpy.py``.""" + import numpy + + C = 2.0 * C1 - 1.0 + S = numpy.sqrt(1 - numpy.square(C)) + F = 2.0 * numpy.pi * F1 + Q = -numpy.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * numpy.sin(F) + output[:, :, 2] = Q * S * numpy.cos(F) + output[:, :, 3] = Q * C diff --git a/pyproject.toml b/pyproject.toml index f6c54ae31ee0..b372c46e580a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,8 @@ requires-python = ">=3.10,<3.15" [project.optional-dependencies] benchmark = [ "asv>=0.6", + # scipy.special.erf is used by the NumPy reference the black_scholes + # benchmark validates its dpnp results against "scipy" ] coverage = [ From 8672e34a52b0090772f2f8ad1384f9cb6df04d54 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Mon, 24 Aug 2026 13:24:26 -0600 Subject: [PATCH 3/5] refactor benchmarks --- CHANGELOG.md | 1 + benchmarks/README.md | 321 ++++++++++++------ benchmarks/asv.conf.json | 4 +- benchmarks/benchmarks/__init__.py | 2 +- .../{benchmark_utils.py => _utils.py} | 37 ++ benchmarks/benchmarks/bench_dpbench.py | 183 +++++----- benchmarks/benchmarks/bench_elementwise.py | 84 ++--- benchmarks/benchmarks/bench_linalg.py | 172 ++-------- benchmarks/benchmarks/bench_random.py | 26 +- benchmarks/benchmarks/common.py | 156 --------- benchmarks/benchmarks/dpbench/README.md | 50 +++ .../benchmarks/dpbench/_dpbench_runner.py | 35 +- benchmarks/pytest_benchmark/README.md | 38 --- benchmarks/pytest_benchmark/test_random.py | 119 ------- pyproject.toml | 7 +- 15 files changed, 522 insertions(+), 713 deletions(-) rename benchmarks/benchmarks/{benchmark_utils.py => _utils.py} (68%) delete mode 100644 benchmarks/benchmarks/common.py create mode 100644 benchmarks/benchmarks/dpbench/README.md delete mode 100644 benchmarks/pytest_benchmark/README.md delete mode 100644 benchmarks/pytest_benchmark/test_random.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 113b0060fdfd..f769cfa48623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ This release is compatible with NumPy 2.5. * Bumped the default minimum required DPC++ compiler version to `2026.1.1` and migrated to the OpenCL ICD loader from the conda-forge `ocl-icd-system` (Linux) and `khronos-opencl-icd-loader` (Windows) packages [#2905](https://github.com/IntelPython/dpnp/pull/2905) * Linked the `dpnp_backend_c` library against only the MKL SYCL domains it uses (`BLAS`, `RNG`, `VM`) [#3012](https://github.com/IntelPython/dpnp/pull/3012) * `dpnp` uses pybind11 3.1.0 [#3015](https://github.com/IntelPython/dpnp/pull/3015) +* Reworked the ASV benchmarks and added end-to-end workload benchmarks derived from dpBench [#2996](https://github.com/IntelPython/dpnp/pull/2996) ### Deprecated diff --git a/benchmarks/README.md b/benchmarks/README.md index f2972cb392ce..3927abb70a7f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,144 +1,267 @@ -# dpnp benchmarks +# dpnp ASV Benchmarks -Benchmarking dpnp using Airspeed Velocity. -Read more about ASV [here](https://asv.readthedocs.io/en/stable/index.html). +Performance benchmarks for [dpnp](https://github.com/IntelPython/dpnp) using +[Airspeed Velocity (ASV)](https://asv.readthedocs.io/en/stable/). -## Usage +## Coverage -Unlike a pure-Python project, dpnp is a SYCL/DPC++ extension that requires the -Intel oneAPI compiler and a lengthy build, so ASV does not build dpnp itself: -`build_command` in `asv.conf.json` is empty and the benchmarks are run against -an **existing environment** that already has dpnp installed. +| File | API | Benchmarks | Params | Sizes | +|------|-----|------------|--------|-------| +| `bench_dpbench.py` | `dpnp` (end-to-end workloads) | `BlackScholes`, `L2Norm`, `PairwiseDistance`, `Rambo`, `Gpairs` | `preset`, `precision` | dpBench presets `S`, `M16Gb`, `M`, `L` | +| `bench_elementwise.py` | `dpnp` vs `numpy` | `Elementwise` (26 unary math functions) | `executor`, `size`, `dtype` | 2^16, 2^20, 2^24 | +| `bench_linalg.py` | `dpnp` vs `numpy` (`dot`, `matmul`, `inner`, `einsum`) | `MatMul` | `executor`, `order`, `dtype` | 16 to 1024 square | +| `bench_random.py` | `dpnp.random` vs `numpy.random` | `Sample` (`rand`, `randn`, `random_sample`) | `executor`, `size` | 2^16, 2^20, 2^24 | -Create an environment -[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html) -and install the benchmarking tooling into it. +### dpBench workloads + +`bench_dpbench.py` runs a set of dpnp workloads derived from +[dpBench](https://github.com/IntelPython/dpbench), which live in +`benchmarks/benchmarks/dpbench/workloads`. They measure the end-to-end time of a +whole workload rather than of an individual API call. + +| Workload | Domain | +| ------------------- | ------------------ | +| `black_scholes` | Finance | +| `l2_norm` | Distance Compute | +| `pairwise_distance` | Distance Compute | +| `rambo` | Particle Physics | +| `gpairs` | Astrophysics | -Install the tooling directly, which leaves the already-built dpnp untouched: +Host input data is generated and copied to the device the way dpBench does, and +each kernel ends with `dpnp.synchronize_array_data`, so a single call blocks +until the device work has finished. dpBench is not a dependency. See +[`benchmarks/dpbench/README.md`](benchmarks/dpbench/README.md) for the +source-to-module mapping and the intended differences. + +## Device and precision + +dpnp allocates on the default SYCL device. Use `ONEAPI_DEVICE_SELECTOR` to +target a specific one: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ + --python=same \ + --launch-method spawn \ + --quick +``` + +**The parameter matrix is the same on every machine.** All four dpBench presets +are declared statically, so a given benchmark has the same parameter set +everywhere and results are comparable across devices and across the CI pool. +What varies per device is which of those points *run*: `setup` calls +`_dpbench_runner.preset_fits` and raises `SkipNotImplemented` for any preset +whose estimated peak element count (the workload's `peak_elements`, taken at the +wider of the two precisions) exceeds **0.25** of the device's `global_mem_size`. +So a large discrete GPU exercises the bigger problem sizes automatically while a +small iGPU reports `S` and skips the rest, and a skipped point stays visible as a +skip rather than vanishing from the matrix. + +The cheapest preset always runs. If even that does not fit, it is attempted +anyway so the failure is a loud allocation error rather than silence. + +Note that dpBench's preset names are not ordered by size: `M16Gb` is *smaller* +than `M` for every workload except `rambo`, where it is larger and equal to `L`. +Anything that needs the cheapest preset sorts explicitly rather than relying on +declaration order. + +**Both precisions are benchmarked.** Devices without fp64 support (common on +iGPUs) skip the `double` points via `SkipNotImplemented` rather than failing, so +such a device still produces `single` results. The `float64` points of +`bench_elementwise.py` and `bench_linalg.py` skip the same way for the `dpnp` +executor; the `numpy` executor is unaffected. dpBench's own configs request +`double` throughout, and that value is kept in each workload's `PRECISION` for +reference. + +No benchmark module opens a SYCL queue at import time, so benchmark discovery +and `asv check` work on a machine with no usable device; only `setup` needs one. + +One caveat on comparability: ASV keys results by machine, commit and +environment, not by device. Benchmarking two devices on the same host therefore +overwrites one set of results with the other. Give each device its own machine +name when you do that: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run --python=same \ + --launch-method spawn --machine "$(hostname)-gpu" +``` + +## Notes on Measurement + +### Process launch method + +**Always pass `--launch-method spawn`.** ASV defaults to a forkserver, which +`fork()`s a process that has already initialized a SYCL runtime; the SYCL +runtime is multi-threaded and not fork-safe, so benchmarks may hang until +`default_benchmark_timeout` expires (reported as `failed`) or fail with +`USM Allocation` errors on `level_zero` devices. `spawn` starts a fresh +interpreter per benchmark and avoids this entirely. + +### Asynchronous execution + +**Every timed body that runs dpnp work must block on it.** dpnp enqueues to a +SYCL queue and returns before the kernel has run, so a body that does not block +measures submission rather than execution. Unsynchronized, a 1024x1024 float32 +`dot` measured **0.4 ms** against **36 ms** synchronized on a CPU device -- which +would have reported dpnp as an order of magnitude faster than NumPy on work +where it is in fact slightly slower. + +The dpBench workloads each end with `dpnp.synchronize_array_data`, and the +comparison suites obtain a synchronizer from `_utils.make_synchronizer` in +`setup` and pass every result through it (`self.sync(...)`). For the `numpy` +executor the synchronizer does nothing. + +### First-call costs + +The first call on a fresh queue pays SYCL kernel/JIT and allocator warmup. +`WorkloadRunner.setup` therefore runs each workload once before ASV starts +timing it, so the dpBench suite is warmed explicitly. The `bench_elementwise.py`, +`bench_linalg.py` and `bench_random.py` suites do **not** warm up and rely on +ASV's default `warmup_time`. + +### Validation + +Each workload ships the NumPy `reference` implementation from dpBench. On the +cheapest preset, `setup` compares the dpnp results for all `OUTPUT_ARGS` +against it (mirroring dpBench's `infrastructure/benchmark_validation.py`, same +`1e-05` relative-error tolerance). A numerically wrong kernel therefore fails +the benchmark instead of being silently timed. Validation runs outside the +timed region and does not affect the reported numbers. + +Only the cheapest preset is validated: the reference runs on the host and at the +larger presets costs far more than the benchmark it guards -- tens of seconds +for `pairwise_distance` at `M16Gb` -- while checking numerics that do not depend +on the problem size. + +### Noise at small presets + +The smallest sizes are dominated by per-call dispatch overhead and are +noticeably noisier. On a CPU device the run-to-run spread of the median at `S` +was measured between **2%** and **25%** across workloads, against the **20%** +`regressions_thresholds` in `asv.conf.json`, whereas the larger presets settled +to a few percent. Treat `S` as a smoke-test size only and do not use it for +regression gating; prefer the largest preset the device fits. + +## Running Benchmarks + +ASV cannot build dpnp -- it is a SYCL/DPC++ extension that requires the Intel +oneAPI compiler and a lengthy build -- so the benchmarks always run against an +**existing environment** that already has dpnp installed. A bare `asv run` is +not supported; always pass `--python=same` or `--environment existing:`. + +Create an environment +[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html), +then install the benchmarking tooling into it: ```bash conda install -c conda-forge asv scipy ``` +`scipy` is needed because `scipy.special.erf` is used by the NumPy reference +that the `black_scholes` benchmark validates its dpnp results against. + Do **not** use `pip install ".[benchmark]"` for an environment that already has dpnp: dpnp is a scikit-build project, so pip reinstalls the `dpnp` package itself and triggers a full oneAPI/DPC++ rebuild of the backend just to pull in -two pure-Python dependencies. The `benchmark` extra exists for the case where -dpnp is being built from source anyway, e.g.: +two pure-Python dependencies. The `benchmark` extra in `pyproject.toml` records +those two dependencies for the case where dpnp is being built from source +anyway; note that the usual editable-install invocation passes `--no-deps`, so +it does *not* install them: ```bash -pip install --no-build-isolation --no-deps -e ".[benchmark]" +pip install --no-build-isolation --no-deps -e . +conda install -c conda-forge asv scipy ``` -Then activate the environment and run the benchmarks against it. The simplest -way is to point ASV at the currently active environment with `--python=same`: +All commands below are run from the `benchmarks/` directory, where +`asv.conf.json` lives. + +Register the machine once. Without this a non-interactive or CI run aborts with +`No information stored about machine`: ```bash -conda activate dpnp_env -asv run --python=same --launch-method spawn --quick HEAD^! +asv machine --yes ``` -Alternatively, point ASV explicitly at an environment's python binary: +Validate the whole suite without running it. This is cheap and catches broken +signatures and import errors; it accepts no `--bench`, so it is all-or-nothing: ```bash -asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ - --launch-method spawn +asv check --python=same ``` -Compare two commits or check for regressions: +Smoke-run the benchmarks, optionally scoped with `--bench`: ```bash -asv continuous --python=same --launch-method spawn HEAD~1 HEAD +asv run --python=same --launch-method spawn --quick --bench bench_dpbench ``` -**Always pass `--launch-method spawn`.** ASV defaults to a forkserver, which -`fork()`s a process that has already initialized a SYCL runtime; the SYCL -runtime is multi-threaded and not fork-safe, so benchmarks may hang until -`default_benchmark_timeout` expires (reported as `failed`) or fail with -`USM Allocation` errors on `level_zero` devices. `spawn` starts a fresh -interpreter per benchmark and avoids this entirely. +This only *prints* results. Without `--set-commit-hash` ASV discards them, so +`asv compare` and `asv publish` will see nothing. -By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR` -environment variable to target a specific device, e.g.: +To record results, assert which revision the installed dpnp corresponds to: ```bash -ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ - --launch-method spawn \ - --python=same +asv run --python=same --launch-method spawn --set-commit-hash HEAD ``` -## Benchmarks +ASV does not verify that claim -- it is your assertion -- and the +`For dpnp commit ...` progress line prints the branch head rather than the +value passed, so trust the result filename or `asv show`. -### `bench_dpbench.py` -- dpBench workloads +Pointing ASV at an interpreter explicitly works too: -`bench_dpbench.py` runs a set of dpnp workloads vendored from -[dpBench](https://github.com/IntelPython/dpbench). The kernels, their data -initialization, and the data-size presets are copied from dpBench and live in -`benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark -class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the -dpBench data-size preset (`S`, `M16Gb`, `M`, `L`) and by floating-point -precision (`single`, `double`). +```bash +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ + --launch-method spawn +``` -Currently vendored workloads: +`asv.conf.json` sets `branches` to `HEAD` rather than to named branches, so that +results recorded on a feature branch are picked up. With named branches +`asv publish` reports `Couldn't find in branches (...)` and silently +drops them. -| Workload | Domain | -| ------------------- | ------------------ | -| `black_scholes` | Finance | -| `l2_norm` | Distance Compute | -| `pairwise_distance` | Distance Compute | -| `rambo` | Particle Physics | -| `gpairs` | Astrophysics | +### Comparing two revisions + +`asv continuous` and any `` range spec cannot be used here: ASV refuses +a range spec when it cannot install the project into the environment. Compare +two recorded runs instead. Rebuild dpnp in the same environment between them, +and omit `--quick` so the statistics path engages: -Host input data is generated and copied to the device exactly the way dpBench -does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call -blocks until the device work has finished. The `time_*` methods invoke the -workload once and let ASV wall-clock-time it (handling repeats, samples and -statistics natively) -- the same end-to-end quantity dpBench itself measures, -and the same plain `time_*` style used by the mkl_fft ASV benchmarks. - -**Precision.** Both `single` and `double` are benchmarked. Devices without fp64 -support (common on iGPUs) skip the `double` parametrization via -`SkipNotImplemented` rather than failing the run, so such a device still -produces `single`-precision results. dpBench's own configs request `double` -throughout; that value is kept in each workload's `PRECISION` for reference. - -**Preset selection.** Presets are chosen per device instead of being hard-coded: -`_dpbench_runner.select_presets` keeps every preset whose estimated peak device -footprint (each workload's `peak_elements`) fits within a fraction of the -device's `global_mem_size`. A large discrete GPU therefore exercises the bigger -problem sizes automatically, while a small iGPU stays on `S`. Note that dpBench's -preset names are not ordered by size -- `M16Gb` is *smaller* than `M`. - -Prefer the largest preset your device fits when looking for regressions. The -smallest sizes are dominated by per-call dispatch overhead and are noticeably -noisier: on a CPU device the run-to-run spread of the median at `S` was measured -at 4-14%, against the 20% `regressions_thresholds` in `asv.conf.json`, whereas -the larger presets settled to a few percent. Timings at `S` are still useful for -a quick smoke test, and ASV's repeat/sample handling absorbs part of the noise. - -**Validation.** Each workload also ships the NumPy `reference` implementation -from dpBench, and every benchmark's `setup` compares the dpnp results for all -`OUTPUT_ARGS` against it (mirroring dpBench's -`infrastructure/benchmark_validation.py`, same `1e-05` relative-error -tolerance). A numerically wrong kernel therefore fails the benchmark instead of -being silently timed. Validation runs outside the timed region and does not -affect the reported numbers. - -### Other benchmark modules - -The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`, -`bench_random.py`) are plain ASV benchmarks comparing dpnp against NumPy. +```bash +# against the old build +asv run --python=same --launch-method spawn --set-commit-hash +# rebuild/reinstall dpnp, then +asv run --python=same --launch-method spawn --set-commit-hash +asv compare +``` + +View recorded results in a browser: + +```bash +asv publish +asv preview +``` ## Writing new benchmarks Read ASV's guidelines for writing benchmarks [here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). -To add another dpBench workload, copy its `_dpnp.py` kernel, -`_numpy.py` reference (as `reference`) and `_initialize.py` -initializer into a new module under `benchmarks/dpbench/workloads`, translate its -`bench_info` TOML presets into the module's `PRESETS` and argument-metadata -constants, add a `peak_elements` estimate (see the existing workloads for the -exact shape), and add the module to `WORKLOADS` in -`benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a -benchmark class for it automatically. +Parameter axes shared by two or more `bench_*` modules live in `_utils.py`; +single-use axes stay in the module that needs them. Two rules keep results +usable: + +* Keep parameter values plain strings, numbers or tuples. A live module or dtype + object renders as `` in the result tables and embeds + a local path in the result identity. +* Keep `params` static. Deriving an axis from the machine makes rows + incomparable between devices; decide per-device behaviour in `setup` instead, + by raising `SkipNotImplemented` (see `bench_dpbench._Workload.setup`). +* Block on dpnp work inside the timed body, or you are timing submission -- see + [Asynchronous execution](#asynchronous-execution). + +To add another dpBench workload, follow +[`benchmarks/dpbench/README.md`](benchmarks/dpbench/README.md), then add a +benchmark class for it to `bench_dpbench.py`. Copy an existing one: it is a +banner, a docstring, a `WORKLOAD` attribute and a one-line `time_*` method -- +the parameter axes are inherited from `_Workload`. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 6741baf28355..0036108a6250 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -2,13 +2,11 @@ "version": 1, "project": "dpnp", "project_url": "https://github.com/IntelPython/dpnp", - "repo": "..", "show_commit_url": "https://github.com/IntelPython/dpnp/commit/", - "build_command": [], + "repo": "..", "branches": [ "HEAD" ], - "dvcs": "git", "environment_type": "conda", "conda_channels": [ "https://software.repos.intel.com/python/conda/", diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py index 75e277849b30..450f408d07fa 100644 --- a/benchmarks/benchmarks/__init__.py +++ b/benchmarks/benchmarks/__init__.py @@ -26,4 +26,4 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -from . import common +"""ASV benchmarks for dpnp.""" diff --git a/benchmarks/benchmarks/benchmark_utils.py b/benchmarks/benchmarks/_utils.py similarity index 68% rename from benchmarks/benchmarks/benchmark_utils.py rename to benchmarks/benchmarks/_utils.py index c095c1a57ab1..8beb7192124c 100644 --- a/benchmarks/benchmarks/benchmark_utils.py +++ b/benchmarks/benchmarks/_utils.py @@ -26,10 +26,47 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** +"""Shared helpers and parameter axes for the dpnp ASV benchmarks.""" + +import dpctl +import numpy from asv_runner.benchmarks.mark import SkipNotImplemented import dpnp +# executor axis, keyed by name so ASV's tables stay readable +_EXECUTORS = {"dpnp": dpnp, "numpy": numpy} +_EXECUTOR_NAMES = list(_EXECUTORS) + +# axes shared across multiple files +_SIZES_1D = [2**16, 2**20, 2**24] +_DTYPES = ["float64", "float32", "int64", "int32"] + +_DEFAULT_QUEUE = None + + +def default_queue(): + """Return a queue on dpnp's default device, created on first use. + + Deferring creation keeps benchmark discovery free of a device requirement. + """ + global _DEFAULT_QUEUE + + if _DEFAULT_QUEUE is None: + _DEFAULT_QUEUE = dpctl.SyclQueue() + return _DEFAULT_QUEUE + + +def make_synchronizer(executor): + """Return a callable blocking until ``executor``'s work has finished. + + dpnp enqueues asynchronously, so a timed body that does not block measures + submission rather than execution. NumPy is synchronous. + """ + if executor == "dpnp": + return dpnp.synchronize_array_data + return lambda result: None + def skip_unsupported_dtype(q, dtype): """Skip the benchmark if the device does not support the given dtype.""" diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py index 2e6218c9d4a6..915288d47a4e 100644 --- a/benchmarks/benchmarks/bench_dpbench.py +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -26,103 +26,130 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""ASV benchmarks for dpnp workloads vendored from dpBench. - -The workloads (kernels + data initialization) and their data-size presets are -copied from dpBench (https://github.com/IntelPython/dpbench); see -``benchmarks/benchmarks/dpbench``. - -Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its output, -so a single call blocks until the device work has finished. The ``time_*`` -methods below simply invoke the workload once and let ASV wall-clock-time it -(handling repeats, samples and statistics natively) -- the same end-to-end -quantity dpBench itself measures, and the same plain ``time_*`` style used by -the mkl_fft ASV benchmarks. - -A separate benchmark class is generated for each workload -- e.g. -``BlackScholes.time_black_scholes`` -- parametrized by the data-size preset and -the floating-point precision. The presets are chosen per device so that only -problem sizes fitting into device memory are benchmarked, and a precision the -device does not support (typically fp64 on an iGPU) is skipped rather than -failing the run. - -``setup`` also validates the dpnp results against the workload's NumPy -reference, so a numerically wrong kernel fails the benchmark instead of being -timed. Validation happens outside the timed region and therefore does not -affect the reported numbers, but it is limited to the cheapest preset: the -reference runs on the host, and at the larger presets it costs far more than -the benchmark it guards (measured at ~70 s for ``pairwise_distance`` at -``M16Gb``) while checking numerics that do not depend on the problem size. +"""Benchmarks for whole dpnp workloads derived from dpBench. + +One class per workload, parametrized by data-size preset and floating-point +precision. See ``dpbench/README.md`` for where the workloads come from. """ -import dpctl +from asv_runner.benchmarks.mark import SkipNotImplemented -from . import benchmark_utils as bench_utils +from ._utils import default_queue, skip_unsupported_dtype from .dpbench import _dpbench_runner as runner -from .dpbench.workloads import WORKLOADS +from .dpbench.workloads import ( + black_scholes, + gpairs, + l2_norm, + pairwise_distance, + rambo, +) + +# Static axes, so the parameter matrix is the same on every machine. What a +# device cannot run is skipped in setup instead. +_PRESETS = ["S", "M16Gb", "M", "L"] +_PRECISIONS = list(runner.PRECISIONS) + + +class _Workload: + """Shared setup for one dpBench-derived workload. + + Subclasses declare ``WORKLOAD`` and a single ``time_*`` method. Defines no + ``time_*`` itself, so ASV does not discover it as a benchmark. + """ + + WORKLOAD = None + params = [_PRESETS, _PRECISIONS] + param_names = ["preset", "precision"] + + def setup(self, preset, precision): + queue = default_queue() + skip_unsupported_dtype(queue, runner.float_dtype(precision)) + + if preset not in self.WORKLOAD.PRESETS: + raise SkipNotImplemented( + f"{self.WORKLOAD.NAME} has no {preset} preset." + ) -# Default-device queue, used to query device capabilities (fp64 support, memory -# size) so the parameter matrix can be tailored to the device. This is the -# device dpnp allocates on by default. -DEVICE_QUEUE = dpctl.SyclQueue() -DEVICE = DEVICE_QUEUE.sycl_device + if not runner.preset_fits(self.WORKLOAD, preset, queue.sycl_device): + raise SkipNotImplemented( + f"Skipping the {preset} preset as its estimated peak footprint" + " does not fit this device's memory." + ) + self._runner = runner.WorkloadRunner(self.WORKLOAD, preset, precision) + self._runner.setup() -def _camel_case(name): - """``black_scholes`` -> ``BlackScholes``, ``l2_norm`` -> ``L2Norm``.""" - return "".join(part.capitalize() for part in name.split("_")) + # Validating the larger presets costs far more than the benchmark it + # guards, and the numerics do not depend on the problem size. + if preset == runner.presets_by_size(self.WORKLOAD)[0]: + self._runner.validate() -def _make_benchmark_class(workload): - """Build an ASV benchmark class for a single dpBench workload.""" +# --------------------------------------------------------------------------- +# Black-Scholes formula (finance) +# --------------------------------------------------------------------------- - class WorkloadBenchmark: - # The per-benchmark timeout is governed by ``default_benchmark_timeout`` - # in ``asv.conf.json``; larger presets on a busy device can take a - # while. - params = [ - runner.select_presets(workload, DEVICE), - list(runner.PRECISIONS), - ] - param_names = ["preset", "precision"] +class BlackScholes(_Workload): + """European option pricing over an array of options.""" - # Preset the results are validated against; see the module docstring. - _validated_preset = runner.presets_by_size(workload)[0] + WORKLOAD = black_scholes + + def time_black_scholes(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# L2 norm (distance compute) +# --------------------------------------------------------------------------- + + +class L2Norm(_Workload): + """Row-wise Euclidean norm of an (npoints, dims) point cloud.""" + + WORKLOAD = l2_norm + + def time_l2_norm(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# Pairwise distance (distance compute) +# --------------------------------------------------------------------------- + + +class PairwiseDistance(_Workload): + """Full (npoints, npoints) Euclidean distance matrix via GEMM.""" + + WORKLOAD = pairwise_distance + + def time_pairwise_distance(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# Rambo (particle physics) +# --------------------------------------------------------------------------- - def setup(self, preset, precision): - # Skip precisions the device does not support (e.g. fp64 on many - # iGPUs), mirroring the dpctl ASV benchmarks. - bench_utils.skip_unsupported_dtype( - DEVICE_QUEUE, runner.float_dtype(precision) - ) - self._runner = runner.WorkloadRunner(workload, preset, precision) - self._runner.setup() - if preset == self._validated_preset: - self._runner.validate() +class Rambo(_Workload): + """Phase-space four-momenta generation for collision events.""" - def time_workload(self, preset, precision): - self._runner.run() + WORKLOAD = rambo - # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``. - WorkloadBenchmark.__name__ = _camel_case(workload.NAME) - WorkloadBenchmark.__qualname__ = WorkloadBenchmark.__name__ + def time_rambo(self, preset, precision): + self._runner.run() - time_method = WorkloadBenchmark.time_workload - time_method.__name__ = f"time_{workload.NAME}" - setattr(WorkloadBenchmark, time_method.__name__, time_method) - del WorkloadBenchmark.time_workload - return WorkloadBenchmark +# --------------------------------------------------------------------------- +# Galaxy pairs (astrophysics) +# --------------------------------------------------------------------------- -def _generate_benchmark_classes(): - """Create and register a benchmark class for every vendored workload.""" - for workload in WORKLOADS: - cls = _make_benchmark_class(workload) - # Register the class at module scope so ASV can discover it. - globals()[cls.__name__] = cls +class Gpairs(_Workload): + """Weighted galaxy-pair counts binned by separation radius.""" + WORKLOAD = gpairs -_generate_benchmark_classes() + def time_gpairs(self, preset, precision): + self._runner.run() diff --git a/benchmarks/benchmarks/bench_elementwise.py b/benchmarks/benchmarks/bench_elementwise.py index 10cd0aea8397..3be1dda3f536 100644 --- a/benchmarks/benchmarks/bench_elementwise.py +++ b/benchmarks/benchmarks/bench_elementwise.py @@ -26,105 +26,107 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for unary elementwise math functions, dpnp against NumPy.""" -import dpnp +from ._utils import ( + _DTYPES, + _EXECUTOR_NAMES, + _EXECUTORS, + _SIZES_1D, + default_queue, + make_synchronizer, + skip_unsupported_dtype, +) -from .common import Benchmark +class Elementwise: + """Unary elementwise ufuncs, dpnp against NumPy.""" -# asv run --python=python --bench Elementwise -# --quick option will run every case once -# but looks like first execution has additional overheads -# (need to be investigated) -class Elementwise(Benchmark): - executors = {"dpnp": dpnp, "numpy": numpy} - params = [ - ["dpnp", "numpy"], - [2**16, 2**20, 2**24], - ["float64", "float32", "int64", "int32"], - ] + params = [_EXECUTOR_NAMES, _SIZES_1D, _DTYPES] param_names = ["executor", "size", "dtype"] def setup(self, executor, size, dtype): - self.np = self.executors[executor] + self.np = _EXECUTORS[executor] + if executor == "dpnp": + skip_unsupported_dtype(default_queue(), dtype) + self.sync = make_synchronizer(executor) dt = getattr(self.np, dtype) self.a = self.np.arange(size, dtype=dt) def time_arccos(self, *args): - self.np.arccos(self.a) + self.sync(self.np.arccos(self.a)) def time_arccosh(self, *args): - self.np.arccosh(self.a) + self.sync(self.np.arccosh(self.a)) def time_arcsin(self, *args): - self.np.arcsin(self.a) + self.sync(self.np.arcsin(self.a)) def time_arcsinh(self, *args): - self.np.arcsinh(self.a) + self.sync(self.np.arcsinh(self.a)) def time_arctan(self, *args): - self.np.arctan(self.a) + self.sync(self.np.arctan(self.a)) def time_arctanh(self, *args): - self.np.arctanh(self.a) + self.sync(self.np.arctanh(self.a)) def time_cbrt(self, *args): - self.np.cbrt(self.a) + self.sync(self.np.cbrt(self.a)) def time_cos(self, *args): - self.np.cos(self.a) + self.sync(self.np.cos(self.a)) def time_cosh(self, *args): - self.np.cosh(self.a) + self.sync(self.np.cosh(self.a)) def time_degrees(self, *args): - self.np.degrees(self.a) + self.sync(self.np.degrees(self.a)) def time_exp(self, *args): - self.np.exp(self.a) + self.sync(self.np.exp(self.a)) def time_exp2(self, *args): - self.np.exp2(self.a) + self.sync(self.np.exp2(self.a)) def time_expm1(self, *args): - self.np.expm1(self.a) + self.sync(self.np.expm1(self.a)) def time_log(self, *args): - self.np.log(self.a) + self.sync(self.np.log(self.a)) def time_log10(self, *args): - self.np.log10(self.a) + self.sync(self.np.log10(self.a)) def time_log1p(self, *args): - self.np.log1p(self.a) + self.sync(self.np.log1p(self.a)) def time_log2(self, *args): - self.np.log2(self.a) + self.sync(self.np.log2(self.a)) def time_rad2deg(self, *args): - self.np.rad2deg(self.a) + self.sync(self.np.rad2deg(self.a)) def time_radians(self, *args): - self.np.radians(self.a) + self.sync(self.np.radians(self.a)) def time_reciprocal(self, *args): - self.np.reciprocal(self.a) + self.sync(self.np.reciprocal(self.a)) def time_sin(self, *args): - self.np.sin(self.a) + self.sync(self.np.sin(self.a)) def time_sinh(self, *args): - self.np.sinh(self.a) + self.sync(self.np.sinh(self.a)) def time_sqrt(self, *args): - self.np.sqrt(self.a) + self.sync(self.np.sqrt(self.a)) def time_square(self, *args): - self.np.square(self.a) + self.sync(self.np.square(self.a)) def time_tan(self, *args): - self.np.tan(self.a) + self.sync(self.np.tan(self.a)) def time_tanh(self, *args): - self.np.tanh(self.a) + self.sync(self.np.tanh(self.a)) diff --git a/benchmarks/benchmarks/bench_linalg.py b/benchmarks/benchmarks/bench_linalg.py index 9d8c08a5e587..e532195652eb 100644 --- a/benchmarks/benchmarks/bench_linalg.py +++ b/benchmarks/benchmarks/bench_linalg.py @@ -26,153 +26,49 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for matrix products, dpnp against NumPy.""" -import dpnp +from ._utils import ( + _DTYPES, + _EXECUTOR_NAMES, + _EXECUTORS, + default_queue, + make_synchronizer, + skip_unsupported_dtype, +) -from .common import TYPES1, Benchmark, get_indexes_rand, get_squares_ +# square matrix orders -- local to this suite +_ORDERS = [16, 32, 64, 128, 256, 512, 1024] -class Eindot(Benchmark): - params = [ - [dpnp, numpy], - [16, 32, 64, 128, 256, 512, 1024], - ["float64", "float32", "int64", "int32"], - ] - param_names = ["executor", "size", "dtype"] +# --------------------------------------------------------------------------- +# Square matrix products +# --------------------------------------------------------------------------- - def setup(self, np, size, dtype): - dt = getattr(np, dtype) - # self.a = np.arange(60000.0).reshape(150, 400) - self.a = np.arange(size * size, dtype=dt).reshape((size, size)) - # self.ac = self.a.copy() - # self.at = self.a.T - # self.atc = self.a.T.copy() - # self.b = np.arange(240000.0).reshape(400, 600) - self.b = np.arange(size * size, dtype=dt).reshape((size, size)) - # self.c = np.arange(600) - # self.d = np.arange(400) +class MatMul: + """Products of two square matrices -- dot, matmul, inner and einsum.""" - # self.a3 = np.arange(480000.).reshape(60, 80, 100) - # self.b3 = np.arange(192000.).reshape(80, 60, 40) + params = [_EXECUTOR_NAMES, _ORDERS, _DTYPES] + param_names = ["executor", "order", "dtype"] - def time_dot_a_b(self, np): - np.dot(self.a, self.b) + def setup(self, executor, order, dtype): + self.np = _EXECUTORS[executor] + self.sync = make_synchronizer(executor) + if executor == "dpnp": + skip_unsupported_dtype(default_queue(), dtype) + dt = getattr(self.np, dtype) + self.a = self.np.arange(order * order, dtype=dt).reshape((order, order)) + self.b = self.np.arange(order * order, dtype=dt).reshape((order, order)) - def time_dot_d_dot_b_c(self, np, *args): - np.dot(self.d, np.dot(self.b, self.c)) + def time_dot(self, executor, order, dtype): + self.sync(self.np.dot(self.a, self.b)) - def time_dot_trans_a_at(self, np, *args): - np.dot(self.a, self.at) + def time_matmul(self, executor, order, dtype): + self.sync(self.np.matmul(self.a, self.b)) - def time_dot_trans_a_atc(self, np, *args): - np.dot(self.a, self.atc) + def time_inner(self, executor, order, dtype): + self.sync(self.np.inner(self.a, self.b)) - def time_dot_trans_at_a(self, np, *args): - np.dot(self.at, self.a) - - def time_dot_trans_atc_a(self, np, *args): - np.dot(self.atc, self.a) - - def time_einsum_i_ij_j(self, np, *args): - np.einsum("i,ij,j", self.d, self.b, self.c) - - def time_einsum_ij_jk_a_b(self, np, *args): - np.einsum("ij,jk", self.a, self.b) - - def time_einsum_ijk_jil_kl(self, np, *args): - np.einsum("ijk,jil->kl", self.a3, self.b3) - - def time_inner_trans_a_a(self, np, *args): - np.inner(self.a, self.a) - - def time_inner_trans_a_ac(self, np, *args): - np.inner(self.a, self.ac) - - def time_matmul_a_b(self, np, *args): - np.matmul(self.a, self.b) - - def time_matmul_d_matmul_b_c(self, np, *args): - np.matmul(self.d, np.matmul(self.b, self.c)) - - def time_matmul_trans_a_at(self, np, *args): - np.matmul(self.a, self.at) - - def time_matmul_trans_a_atc(self, np, *args): - np.matmul(self.a, self.atc) - - def time_matmul_trans_at_a(self, np, *args): - np.matmul(self.at, self.a) - - def time_matmul_trans_atc_a(self, np, *args): - np.matmul(self.atc, self.a) - - def time_tensordot_a_b_axes_1_0_0_1(self, np, *args): - np.tensordot(self.a3, self.b3, axes=([1, 0], [0, 1])) - - -class Linalg(Benchmark): - params = [[dpnp, numpy], ["svd", "pinv", "det", "norm"], TYPES1] - param_names = ["executor", "op", "type"] - - def setup(self, np, op, typename): - np.seterr(all="ignore") - - self.func = getattr(np.linalg, op) - - if op == "cholesky": - # we need a positive definite - self.a = np.dot( - get_squares_()[typename], get_squares_()[typename].T - ) - else: - self.a = get_squares_()[typename] - - # check that dtype is supported at all - try: - self.func(self.a[:2, :2]) - except TypeError: - raise NotImplementedError() - - def time_op(self, np, op, typename): - self.func(self.a) - - -class Lstsq(Benchmark): - params = [dpnp, numpy] - param_names = ["executor"] - - def setup(self, np): - self.a = get_squares_()["float64"] - self.b = get_indexes_rand()[:100].astype(np.float64) - - def time_numpy_linalg_lstsq_a__b_float64(self, np): - np.linalg.lstsq(self.a, self.b, rcond=-1) - - -# class Einsum(Benchmark): -# param_names = ['dtype'] -# params = [[np.float64]] -# def setup(self, dtype): -# self.a = np.arange(2900, dtype=dtype) -# self.b = np.arange(3000, dtype=dtype) -# self.c = np.arange(24000, dtype=dtype).reshape(20, 30, 40) -# self.c1 = np.arange(1200, dtype=dtype).reshape(30, 40) -# self.d = np.arange(10000, dtype=dtype).reshape(10,100,10) - -# #outer(a,b): trigger sum_of_products_contig_stride0_outcontig_two -# def time_einsum_outer(self, dtype): -# np.einsum("i,j", self.a, self.b, optimize=True) - -# # multiply(a, b):trigger sum_of_products_contig_two -# def time_einsum_multiply(self, dtype): -# np.einsum("..., ...", self.c1, self.c , optimize=True) - -# # sum and multiply:trigger sum_of_products_contig_stride0_outstride0_two -# def time_einsum_sum_mul(self, dtype): -# np.einsum(",i...->", 300, self.d, optimize=True) - -# # sum and multiply:trigger sum_of_products_stride0_contig_outstride0_two -# def time_einsum_sum_mul2(self, dtype): -# np.einsum("i...,->", self.d, 300, optimize=True) + def time_einsum_ij_jk(self, executor, order, dtype): + self.sync(self.np.einsum("ij,jk", self.a, self.b)) diff --git a/benchmarks/benchmarks/bench_random.py b/benchmarks/benchmarks/bench_random.py index 191569842371..29ac0ee5e82d 100644 --- a/benchmarks/benchmarks/bench_random.py +++ b/benchmarks/benchmarks/bench_random.py @@ -26,30 +26,34 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for random sampling, dpnp.random against numpy.random.""" -import dpnp +from ._utils import ( + _EXECUTOR_NAMES, + _EXECUTORS, + _SIZES_1D, + make_synchronizer, +) -from .common import Benchmark +class Sample: + """Random sampling, dpnp against NumPy.""" -# asv run --python=python --quick --bench Sample -class Sample(Benchmark): - executors = {"dpnp": dpnp, "numpy": numpy} - params = [["dpnp", "numpy"], [2**16, 2**20, 2**24]] + params = [_EXECUTOR_NAMES, _SIZES_1D] param_names = ["executor", "size"] def setup(self, executor, size): - self.executor = self.executors[executor] + self.executor = _EXECUTORS[executor] + self.sync = make_synchronizer(executor) def time_rand(self, executor, size): np = self.executor - np.random.rand(size) + self.sync(np.random.rand(size)) def time_randn(self, executor, size): np = self.executor - np.random.randn(size) + self.sync(np.random.randn(size)) def time_random_sample(self, executor, size): np = self.executor - np.random.random_sample((size,)) + self.sync(np.random.random_sample((size,))) diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py deleted file mode 100644 index 0d708c2789d1..000000000000 --- a/benchmarks/benchmarks/common.py +++ /dev/null @@ -1,156 +0,0 @@ -# ***************************************************************************** -# Copyright (c) 2020, Intel Corporation -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# - Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# - Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -# THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - -import random - -import numpy - -# Various pre-crafted datasets/variables for testing -# !!! Must not be changed -- only appended !!! -# while testing numpy we better not rely on numpy to produce random -# sequences -random.seed(1) -# but will seed it nevertheless -numpy.random.seed(1) - -nx, ny = 1000, 1000 -# reduced squares based on indexes_rand, primarily for testing more -# time-consuming functions (ufunc, linalg, etc) -nxs, nys = 100, 100 - -# a set of interesting types to test -# NOTE: extended-precision types (numpy.longdouble / numpy.complex256, and the -# removed numpy.longfloat alias) are intentionally absent -- dpnp has no -# counterpart for them, so dpnp.asarray() rejects such input. -TYPES1 = [ - "int16", - "float16", - "int32", - "float32", - "int64", - "float64", - "complex64", - "complex128", -] - - -def memoize(func): - result = [] - - def wrapper(): - if not result: - result.append(func()) - return result[0] - - return wrapper - - -# values which will be used to construct our sample data matrices -# replicate 10 times to speed up initial imports of this helper -# and generate some redundancy - - -@memoize -def get_values(): - rnd = numpy.random.RandomState(1) - values = numpy.tile(rnd.uniform(0, 100, size=nx * ny // 10), 10) - return values - - -@memoize -def get_squares(): - values = get_values() - squares = { - t: numpy.array(values, dtype=getattr(numpy, t)).reshape((nx, ny)) - for t in TYPES1 - } - - # adjust complex ones to have non-degenerated imagery part -- use - # original data transposed for that - for t, v in squares.items(): - if t.startswith("complex"): - v += v.T * 1j - return squares - - -@memoize -def get_squares_(): - # smaller squares - squares_ = {t: s[:nxs, :nys] for t, s in get_squares().items()} - return squares_ - - -@memoize -def get_vectors(): - # vectors - vectors = {t: s[0] for t, s in get_squares().items()} - return vectors - - -@memoize -def get_indexes(): - indexes = list(range(nx)) - # so we do not have all items - indexes.pop(5) - indexes.pop(95) - - indexes = numpy.array(indexes) - return indexes - - -@memoize -def get_indexes_rand(): - rnd = random.Random(1) - - indexes_rand = get_indexes().tolist() # copy - rnd.shuffle(indexes_rand) # in-place shuffle - indexes_rand = numpy.array(indexes_rand) - return indexes_rand - - -@memoize -def get_indexes_(): - # smaller versions - indexes = get_indexes() - indexes_ = indexes[indexes < nxs] - return indexes_ - - -@memoize -def get_indexes_rand_(): - indexes_rand = get_indexes_rand() - indexes_rand_ = indexes_rand[indexes_rand < nxs] - return indexes_rand_ - - -class Benchmark: - # warmup_time = 0 - # number = 3 # test repeats for one setup - # repeat = 1 - # rounds = 1 - pass diff --git a/benchmarks/benchmarks/dpbench/README.md b/benchmarks/benchmarks/dpbench/README.md new file mode 100644 index 000000000000..ca9afef6717f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/README.md @@ -0,0 +1,50 @@ +## dpBench-derived workloads + +The modules under `workloads/` reproduce benchmarks from +[dpBench](https://github.com/IntelPython/dpbench), so that dpnp is measured on +the same quantity: the end-to-end time of a whole workload rather than of a +single API call. dpBench is not a dependency; `_dpbench_runner.py` re-implements +the parts ASV needs (data initialization, host-to-device transfer, execution and +reference validation). + +Reference version: dpBench `0.2.0+79.g4501644`. + +Per workload, three modules from `dpbench/benchmarks/default//` and one +config from `dpbench/configs/bench_info/` map onto one module here: + +| dpnp module | dpBench sources | +| -------------------------------- | ----------------------------------------------------------------------------------- | +| `workloads/black_scholes.py` | `black_scholes_{dpnp,numpy,initialize}.py`, `black_scholes.toml` | +| `workloads/l2_norm.py` | `l2_norm_{dpnp,numpy,initialize}.py`, `l2_norm.toml` | +| `workloads/pairwise_distance.py` | `pairwise_distance_{dpnp,numpy,initialize}.py`, `pairwise_distance.toml` | +| `workloads/rambo.py` | `rambo_{dpnp,numpy,initialize}.py`, `rambo.toml` | +| `workloads/gpairs.py` | `gpairs_{dpnp,numpy,initialize}.py`, `gpairs.toml` | + +`_dpnp.py` became `()`, `_numpy.py` became `reference()` and +`_initialize.py` became `initialize()`. From the TOML, `[benchmark]` gives +`INPUT_ARGS` / `ARRAY_ARGS` / `OUTPUT_ARGS`, `[benchmark.init]` gives +`INIT_INPUT_ARGS` / `INIT_OUTPUT_ARGS` / `PRECISION`, and +`[benchmark.parameters.*]` gives `PRESETS`. + +### Intended differences + +1. `black_scholes` calls `dpnp.scipy.special.erf`, where dpnp now keeps `erf`. +2. Every kernel ends with `dpnp.synchronize_array_data()`. ASV times the + `time_*` method directly, so the kernel has to block or only host-side + dispatch is measured. +3. `rambo.initialize` draws its random block in one `numpy.random.rand` call + rather than element by element. This consumes the same RNG stream in the same + order, so the data is bit-identical, but it is far faster -- which matters + because ASV re-runs `setup` for every round. +4. `peak_elements(params)` is new: it estimates a preset's peak element count so + `_dpbench_runner.preset_fits` can skip presets too large for the device. + +### Adding a workload + +Add a module under `workloads/` exposing the same interface as the existing ones, +translate its `bench_info` TOML into the metadata constants, add a +`peak_elements` estimate, and record it in the table above. + +Then add a benchmark class to `bench_dpbench.py` -- that is what puts the +workload into the suite. `WORKLOADS` in `workloads/__init__.py` is only a +registry; adding to it alone has no effect. diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py index ffd6b7d6e12b..c3dfed4f9e57 100644 --- a/benchmarks/benchmarks/dpbench/_dpbench_runner.py +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -89,32 +89,21 @@ def float_dtype(precision): return build_types_dict(precision)["float"] -def select_presets(workload, device, precision="double"): - """Pick the dpBench presets that fit into ``device``'s global memory. - - dpBench leaves preset selection to the user; ASV needs it decided up front - because ``params`` is evaluated at import time. Every preset whose - estimated peak footprint (see each workload's ``peak_elements``) fits the - memory budget is returned, so a large discrete GPU automatically exercises - the bigger problem sizes while a small iGPU stays on ``S``. - - ``precision`` is deliberately the *widest* precision benchmarked rather - than each one separately: it keeps the preset list identical across the - precision parameter, so ASV's parameter matrix stays rectangular and - results remain comparable. +def preset_fits(workload, preset, device, precision="double"): + """Whether ``preset``'s estimated peak footprint fits ``device``'s memory. + + ``precision`` is the *widest* precision benchmarked, so the verdict is the + same for every precision parameter. """ + # The cheapest preset always runs, so that an undersized device fails loudly + # on allocation rather than reporting nothing. + if preset == presets_by_size(workload)[0]: + return True + itemsize = float_dtype(precision).itemsize budget = _MEMORY_BUDGET_FRACTION * device.global_mem_size - - fitting = [ - name - for name in presets_by_size(workload) - if workload.peak_elements(workload.PRESETS[name]) * itemsize <= budget - ] - # Always benchmark something: if even the smallest preset is over budget, - # fall back to it and let the run fail loudly on allocation instead of - # silently reporting no data at all. - return fitting or presets_by_size(workload)[:1] + peak = workload.peak_elements(workload.PRESETS[preset]) + return peak * itemsize <= budget def presets_by_size(workload): diff --git a/benchmarks/pytest_benchmark/README.md b/benchmarks/pytest_benchmark/README.md deleted file mode 100644 index 77015a089ef9..000000000000 --- a/benchmarks/pytest_benchmark/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# dpnp/benchmarks/pytest_benchmark/ - -## Prerequisites -* pytest >= 6.1.1 -* pytest-benchmark >= 3.4.1 - - -## Running benchmark tests -```bash -pytest benchmarks/ --benchmark-json=results.json -``` -Running tests and saving the current run into `STORAGE`, see [1] -```bash -pytest benchmarks/ --benchmark-json=results.json --benchmark-autosave -``` - -## Creating `.csv` report -```bash -pytest-benchmark compare results.json --csv=results.csv --group-by='name' -``` - -## Optional: creating histogram -Note: make sure that `pytest-benchmark[histogram]` installed -```bash -# example -pip install pytest-benchmark[histogram] -pytest -vv benchmarks/ --benchmark-autosave --benchmark-histogram -pytest-benchmark compare .benchmarks/Linux-CPython-3.7-64bit/* --histogram -``` - -## Advanced running example -``` -pytest benchmarks/ --benchmark-columns='min, max, mean, stddev, median, rounds, iterations' --benchmark-json=results.json --benchmark-autosave -pytest-benchmark compare results.json --csv=results.csv --group-by='name' -``` - - -[1] https://pytest-benchmark.readthedocs.io/en/latest/usage.html diff --git a/benchmarks/pytest_benchmark/test_random.py b/benchmarks/pytest_benchmark/test_random.py deleted file mode 100644 index 5c91894b2480..000000000000 --- a/benchmarks/pytest_benchmark/test_random.py +++ /dev/null @@ -1,119 +0,0 @@ -# cython: language_level=3 -# ***************************************************************************** -# Copyright (c) 2016, Intel Corporation -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# - Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# - Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -# THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - -import numpy as np -import pytest - -import dpnp - -ROUNDS = 30 -ITERATIONS = 4 - -NNUMBERS = 2**26 - - -@pytest.mark.parametrize( - "function", [dpnp.random.beta, np.random.beta], ids=["dpnp", "numpy"] -) -def test_beta(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 4.0, - 5.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", - [dpnp.random.exponential, np.random.exponential], - ids=["dpnp", "numpy"], -) -def test_exponential(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 4.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.gamma, np.random.gamma], ids=["dpnp", "numpy"] -) -def test_gamma(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 2.0, - 4.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.normal, np.random.normal], ids=["dpnp", "numpy"] -) -def test_normal(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 0.0, - 1.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.uniform, np.random.uniform], ids=["dpnp", "numpy"] -) -def test_uniform(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 0.0, - 1.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) diff --git a/pyproject.toml b/pyproject.toml index 60220ea811c8..0cf3e9721872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,12 +75,7 @@ readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10,<3.15" [project.optional-dependencies] -benchmark = [ - "asv>=0.6", - # scipy.special.erf is used by the NumPy reference the black_scholes - # benchmark validates its dpnp results against - "scipy" -] +benchmark = ["asv>=0.6", "scipy"] coverage = [ "coverage", "Cython", From 4fe4eba40f01d5200599b101d328a80c890984b8 Mon Sep 17 00:00:00 2001 From: Jordan Harlow <109105754+jharlow-intel@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:41:12 -0600 Subject: [PATCH 4/5] fix: branches for ASV --- benchmarks/asv.conf.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 0036108a6250..7310fc5ceb7b 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -5,7 +5,8 @@ "show_commit_url": "https://github.com/IntelPython/dpnp/commit/", "repo": "..", "branches": [ - "HEAD" + "master", + "dev-milestone" ], "environment_type": "conda", "conda_channels": [ From a97591d71fff54a3841047a638d73e95812da6e6 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Tue, 25 Aug 2026 19:14:50 -0600 Subject: [PATCH 5/5] fix: more review --- benchmarks/README.md | 48 +++-- benchmarks/benchmarks/_utils.py | 13 +- benchmarks/benchmarks/bench_dpbench.py | 3 +- benchmarks/benchmarks/bench_elementwise.py | 195 +++++++++++------- benchmarks/benchmarks/bench_linalg.py | 64 +++++- benchmarks/benchmarks/bench_random.py | 2 + benchmarks/benchmarks/dpbench/README.md | 2 +- .../benchmarks/dpbench/_dpbench_runner.py | 31 +-- benchmarks/requirements.txt | 7 + 9 files changed, 249 insertions(+), 116 deletions(-) create mode 100644 benchmarks/requirements.txt diff --git a/benchmarks/README.md b/benchmarks/README.md index 3927abb70a7f..0cdeb0e108f6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -8,8 +8,9 @@ Performance benchmarks for [dpnp](https://github.com/IntelPython/dpnp) using | File | API | Benchmarks | Params | Sizes | |------|-----|------------|--------|-------| | `bench_dpbench.py` | `dpnp` (end-to-end workloads) | `BlackScholes`, `L2Norm`, `PairwiseDistance`, `Rambo`, `Gpairs` | `preset`, `precision` | dpBench presets `S`, `M16Gb`, `M`, `L` | -| `bench_elementwise.py` | `dpnp` vs `numpy` | `Elementwise` (26 unary math functions) | `executor`, `size`, `dtype` | 2^16, 2^20, 2^24 | -| `bench_linalg.py` | `dpnp` vs `numpy` (`dot`, `matmul`, `inner`, `einsum`) | `MatMul` | `executor`, `order`, `dtype` | 16 to 1024 square | +| `bench_elementwise.py` | `dpnp` vs `numpy` | `Unary` (31 ufuncs), `Binary` (7 ufuncs) | `executor`, `ufunc`, `size`, `dtype` (float only) | 2^16, 2^20, 2^24 | +| `bench_linalg.py` | `dpnp` vs `numpy` (`dot`, `matmul`, `inner`, `einsum`; contiguous and transposed) | `MatMul` | `executor`, `order`, `dtype` (float and int) | 16 to 1024 square | +| `bench_linalg.py` | `dpnp.linalg` vs `numpy.linalg` (`det`, `norm`, `solve`, `svd`) | `Linalg` | `executor`, `order`, `dtype` (float only) | 16 to 1024 square | | `bench_random.py` | `dpnp.random` vs `numpy.random` | `Sample` (`rand`, `randn`, `random_sample`) | `executor`, `size` | 2^16, 2^20, 2^24 | ### dpBench workloads @@ -50,11 +51,12 @@ are declared statically, so a given benchmark has the same parameter set everywhere and results are comparable across devices and across the CI pool. What varies per device is which of those points *run*: `setup` calls `_dpbench_runner.preset_fits` and raises `SkipNotImplemented` for any preset -whose estimated peak element count (the workload's `peak_elements`, taken at the -wider of the two precisions) exceeds **0.25** of the device's `global_mem_size`. -So a large discrete GPU exercises the bigger problem sizes automatically while a -small iGPU reports `S` and skips the rest, and a skipped point stays visible as a -skip rather than vanishing from the matrix. +whose estimated peak footprint -- the workload's `peak_elements` at the point's +own precision -- exceeds **0.25** of the device's `global_mem_size`. So a large +discrete GPU exercises the bigger problem sizes automatically while a small iGPU +reports `S` and skips the rest, and a skipped point stays visible as a skip +rather than vanishing from the matrix. Since `single` needs half the memory of +`double`, it reaches one preset further on a given device. The cheapest preset always runs. If even that does not fit, it is attempted anyway so the failure is a loud allocation error rather than silence. @@ -100,10 +102,10 @@ interpreter per benchmark and avoids this entirely. **Every timed body that runs dpnp work must block on it.** dpnp enqueues to a SYCL queue and returns before the kernel has run, so a body that does not block -measures submission rather than execution. Unsynchronized, a 1024x1024 float32 -`dot` measured **0.4 ms** against **36 ms** synchronized on a CPU device -- which -would have reported dpnp as an order of magnitude faster than NumPy on work -where it is in fact slightly slower. +measures submission rather than execution. On a CPU device a 1024x1024 float32 +`dot` measured **0.3 ms** unsynchronized against **18 ms** synchronized -- which +would have reported dpnp as far faster than NumPy's **10 ms** on work where it is +in fact 1.7x slower. The dpBench workloads each end with `dpnp.synchronize_array_data`, and the comparison suites obtain a synchronizer from `_utils.make_synchronizer` in @@ -115,8 +117,13 @@ executor the synchronizer does nothing. The first call on a fresh queue pays SYCL kernel/JIT and allocator warmup. `WorkloadRunner.setup` therefore runs each workload once before ASV starts timing it, so the dpBench suite is warmed explicitly. The `bench_elementwise.py`, -`bench_linalg.py` and `bench_random.py` suites do **not** warm up and rely on -ASV's default `warmup_time`. +`bench_linalg.py` and `bench_random.py` suites each run their operation once in +`setup` as well, so a `--quick` measurement is not dominated by one-time cost. + +`--quick` still takes a single sample, so use it to check that benchmarks run +rather than to compare them: before the explicit warmups a `--quick` +`dot(a, a.T)` measured 2.6x its contiguous counterpart, where a repeated +measurement puts it at 0.7x. ### Validation @@ -157,7 +164,10 @@ conda install -c conda-forge asv scipy ``` `scipy` is needed because `scipy.special.erf` is used by the NumPy reference -that the `black_scholes` benchmark validates its dpnp results against. +that the `black_scholes` benchmark validates its dpnp results against. It is not +a dpnp runtime dependency, so it also has to be listed in `requirements.txt`, +which is what CI installs into the benchmarking environment. Keep that file and +the `benchmark` extra in `pyproject.toml` in step. Do **not** use `pip install ".[benchmark]"` for an environment that already has dpnp: dpnp is a scikit-build project, so pip reinstalls the `dpnp` package @@ -215,11 +225,6 @@ asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ --launch-method spawn ``` -`asv.conf.json` sets `branches` to `HEAD` rather than to named branches, so that -results recorded on a feature branch are picked up. With named branches -`asv publish` reports `Couldn't find in branches (...)` and silently -drops them. - ### Comparing two revisions `asv continuous` and any `` range spec cannot be used here: ASV refuses @@ -242,6 +247,11 @@ asv publish asv preview ``` +The published dashboard only covers the branches listed in `asv.conf.json` +(`master` and `dev-milestone`). Results recorded for a commit on any other +branch are dropped with `Couldn't find in branches (...)`, so use +`asv compare` for feature-branch and PR work -- it does not consult `branches`. + ## Writing new benchmarks Read ASV's guidelines for writing benchmarks diff --git a/benchmarks/benchmarks/_utils.py b/benchmarks/benchmarks/_utils.py index 8beb7192124c..9c1fd84c339f 100644 --- a/benchmarks/benchmarks/_utils.py +++ b/benchmarks/benchmarks/_utils.py @@ -40,7 +40,6 @@ # axes shared across multiple files _SIZES_1D = [2**16, 2**20, 2**24] -_DTYPES = ["float64", "float32", "int64", "int32"] _DEFAULT_QUEUE = None @@ -63,9 +62,15 @@ def make_synchronizer(executor): dpnp enqueues asynchronously, so a timed body that does not block measures submission rather than execution. NumPy is synchronous. """ - if executor == "dpnp": - return dpnp.synchronize_array_data - return lambda result: None + if executor != "dpnp": + return lambda result: None + + def sync(result): + # Some results are tuples, e.g. linalg.svd. + for array in result if isinstance(result, tuple) else (result,): + dpnp.synchronize_array_data(array) + + return sync def skip_unsupported_dtype(q, dtype): diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py index 915288d47a4e..f7395578317e 100644 --- a/benchmarks/benchmarks/bench_dpbench.py +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -70,7 +70,8 @@ def setup(self, preset, precision): f"{self.WORKLOAD.NAME} has no {preset} preset." ) - if not runner.preset_fits(self.WORKLOAD, preset, queue.sycl_device): + device = queue.sycl_device + if not runner.preset_fits(self.WORKLOAD, preset, device, precision): raise SkipNotImplemented( f"Skipping the {preset} preset as its estimated peak footprint" " does not fit this device's memory." diff --git a/benchmarks/benchmarks/bench_elementwise.py b/benchmarks/benchmarks/bench_elementwise.py index 3be1dda3f536..4223ec4e6e3f 100644 --- a/benchmarks/benchmarks/bench_elementwise.py +++ b/benchmarks/benchmarks/bench_elementwise.py @@ -26,10 +26,14 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""Benchmarks for unary elementwise math functions, dpnp against NumPy.""" +"""Benchmarks for elementwise ufuncs, dpnp against NumPy. + +The ufunc is a parameter rather than a method per function, following NumPy's +own ``benchmarks/benchmarks/bench_ufunc.py``, so extending the coverage is a +one-line change. +""" from ._utils import ( - _DTYPES, _EXECUTOR_NAMES, _EXECUTORS, _SIZES_1D, @@ -38,95 +42,136 @@ skip_unsupported_dtype, ) - -class Elementwise: - """Unary elementwise ufuncs, dpnp against NumPy.""" - - params = [_EXECUTOR_NAMES, _SIZES_1D, _DTYPES] - param_names = ["executor", "size", "dtype"] - - def setup(self, executor, size, dtype): +# Float only. These ufuncs return floats, so an integer input would time the +# int-to-float promotion rather than the kernel, and would not be comparable +# with the float cells. NumPy's own suite restricts them the same way. +_FLOAT_DTYPES = ["float64", "float32"] + +# Only one name per ufunc: rad2deg/deg2rad wrap the same backend functions as +# degrees/radians, and abs, true_divide and pow are aliases of absolute, divide +# and power. +_UNARY = [ + "absolute", + "arccos", + "arccosh", + "arcsin", + "arcsinh", + "arctan", + "arctanh", + "cbrt", + "ceil", + "cos", + "cosh", + "degrees", + "exp", + "exp2", + "expm1", + "floor", + "log", + "log10", + "log1p", + "log2", + "radians", + "reciprocal", + "rint", + "sign", + "sin", + "sinh", + "sqrt", + "square", + "tan", + "tanh", + "trunc", +] + +_BINARY = [ + "add", + "arctan2", + "divide", + "hypot", + "multiply", + "power", + "subtract", +] + +# Input ranges keeping each ufunc inside its domain, so that none is timed +# entirely on an out-of-domain path. +_RANGES = { + "arccos": (-1, 1), + "arccosh": (1, 10), + "arcsin": (-1, 1), + "arctanh": (-0.9, 0.9), + "log": (1, 10), + "log10": (1, 10), + "log1p": (1, 10), + "log2": (1, 10), + "reciprocal": (1, 10), + "sqrt": (1, 10), +} +_DEFAULT_RANGE = (-10, 10) + +# Positive first operand and a small second one, so divide never sees a zero +# and power stays in range. +_BINARY_RANGES = ((1, 10), (1, 2)) + + +class _Ufunc: + """Shared setup for a ufunc benchmark. + + Defines no ``time_*`` method, so ASV does not discover it as a benchmark. + """ + + param_names = ["executor", "ufunc", "size", "dtype"] + + def setup(self, executor, ufunc, size, dtype): self.np = _EXECUTORS[executor] if executor == "dpnp": skip_unsupported_dtype(default_queue(), dtype) self.sync = make_synchronizer(executor) - dt = getattr(self.np, dtype) - self.a = self.np.arange(size, dtype=dt) - - def time_arccos(self, *args): - self.sync(self.np.arccos(self.a)) - - def time_arccosh(self, *args): - self.sync(self.np.arccosh(self.a)) - - def time_arcsin(self, *args): - self.sync(self.np.arcsin(self.a)) - - def time_arcsinh(self, *args): - self.sync(self.np.arcsinh(self.a)) - - def time_arctan(self, *args): - self.sync(self.np.arctan(self.a)) - - def time_arctanh(self, *args): - self.sync(self.np.arctanh(self.a)) - - def time_cbrt(self, *args): - self.sync(self.np.cbrt(self.a)) - - def time_cos(self, *args): - self.sync(self.np.cos(self.a)) - - def time_cosh(self, *args): - self.sync(self.np.cosh(self.a)) - - def time_degrees(self, *args): - self.sync(self.np.degrees(self.a)) - - def time_exp(self, *args): - self.sync(self.np.exp(self.a)) + self.fn = getattr(self.np, ufunc) - def time_exp2(self, *args): - self.sync(self.np.exp2(self.a)) + def _input(self, size, dtype, bounds): + lo, hi = bounds + return self.np.linspace(lo, hi, size, dtype=getattr(self.np, dtype)) - def time_expm1(self, *args): - self.sync(self.np.expm1(self.a)) - def time_log(self, *args): - self.sync(self.np.log(self.a)) +# --------------------------------------------------------------------------- +# One input -- transcendental, rounding and sign ufuncs +# --------------------------------------------------------------------------- - def time_log10(self, *args): - self.sync(self.np.log10(self.a)) - def time_log1p(self, *args): - self.sync(self.np.log1p(self.a)) +class Unary(_Ufunc): + """Unary ufuncs, e.g. exp, sqrt, floor.""" - def time_log2(self, *args): - self.sync(self.np.log2(self.a)) + params = [_EXECUTOR_NAMES, _UNARY, _SIZES_1D, _FLOAT_DTYPES] - def time_rad2deg(self, *args): - self.sync(self.np.rad2deg(self.a)) + def setup(self, executor, ufunc, size, dtype): + super().setup(executor, ufunc, size, dtype) + bounds = _RANGES.get(ufunc, _DEFAULT_RANGE) + self.a = self._input(size, dtype, bounds) + # Warm up, so the first timed call does not pay device setup. + self.sync(self.fn(self.a)) - def time_radians(self, *args): - self.sync(self.np.radians(self.a)) + def time_unary(self, executor, ufunc, size, dtype): + self.sync(self.fn(self.a)) - def time_reciprocal(self, *args): - self.sync(self.np.reciprocal(self.a)) - def time_sin(self, *args): - self.sync(self.np.sin(self.a)) +# --------------------------------------------------------------------------- +# Two inputs -- arithmetic ufuncs +# --------------------------------------------------------------------------- - def time_sinh(self, *args): - self.sync(self.np.sinh(self.a)) - def time_sqrt(self, *args): - self.sync(self.np.sqrt(self.a)) +class Binary(_Ufunc): + """Binary ufuncs, e.g. add, power, hypot.""" - def time_square(self, *args): - self.sync(self.np.square(self.a)) + params = [_EXECUTOR_NAMES, _BINARY, _SIZES_1D, _FLOAT_DTYPES] - def time_tan(self, *args): - self.sync(self.np.tan(self.a)) + def setup(self, executor, ufunc, size, dtype): + super().setup(executor, ufunc, size, dtype) + first, second = _BINARY_RANGES + self.a = self._input(size, dtype, first) + self.b = self._input(size, dtype, second) + self.sync(self.fn(self.a, self.b)) - def time_tanh(self, *args): - self.sync(self.np.tanh(self.a)) + def time_binary(self, executor, ufunc, size, dtype): + self.sync(self.fn(self.a, self.b)) diff --git a/benchmarks/benchmarks/bench_linalg.py b/benchmarks/benchmarks/bench_linalg.py index e532195652eb..4fc539b2eb54 100644 --- a/benchmarks/benchmarks/bench_linalg.py +++ b/benchmarks/benchmarks/bench_linalg.py @@ -26,10 +26,9 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""Benchmarks for matrix products, dpnp against NumPy.""" +"""Benchmarks for matrix products and decompositions, dpnp against NumPy.""" from ._utils import ( - _DTYPES, _EXECUTOR_NAMES, _EXECUTORS, default_queue, @@ -40,6 +39,13 @@ # square matrix orders -- local to this suite _ORDERS = [16, 32, 64, 128, 256, 512, 1024] +# Integers are native for the matrix products, which do not promote. +_DTYPES = ["float64", "float32", "int64", "int32"] + +# LAPACK has no integer path, so an integer input to a decomposition would time +# the promotion instead of the factorization. +_FLOAT_DTYPES = ["float64", "float32"] + # --------------------------------------------------------------------------- # Square matrix products @@ -60,15 +66,69 @@ def setup(self, executor, order, dtype): dt = getattr(self.np, dtype) self.a = self.np.arange(order * order, dtype=dt).reshape((order, order)) self.b = self.np.arange(order * order, dtype=dt).reshape((order, order)) + # Non-contiguous operand, which reaches a different BLAS path: the + # transpose is expressed as a flag rather than as a copy. + self.at = self.a.T + # Pay the one-time SYCL and oneMKL initialization before timing. + self.sync(self.np.dot(self.a, self.b)) def time_dot(self, executor, order, dtype): self.sync(self.np.dot(self.a, self.b)) + def time_dot_transposed(self, executor, order, dtype): + self.sync(self.np.dot(self.a, self.at)) + def time_matmul(self, executor, order, dtype): self.sync(self.np.matmul(self.a, self.b)) + def time_matmul_transposed(self, executor, order, dtype): + self.sync(self.np.matmul(self.a, self.at)) + def time_inner(self, executor, order, dtype): self.sync(self.np.inner(self.a, self.b)) def time_einsum_ij_jk(self, executor, order, dtype): self.sync(self.np.einsum("ij,jk", self.a, self.b)) + + +# --------------------------------------------------------------------------- +# LAPACK-backed decompositions and norms +# --------------------------------------------------------------------------- + + +class Linalg: + """Square-matrix decompositions -- det, norm, solve and svd.""" + + params = [_EXECUTOR_NAMES, _ORDERS, _FLOAT_DTYPES] + param_names = ["executor", "order", "dtype"] + + def setup(self, executor, order, dtype): + self.np = _EXECUTORS[executor] + self.sync = make_synchronizer(executor) + if executor == "dpnp": + skip_unsupported_dtype(default_queue(), dtype) + dt = getattr(self.np, dtype) + # I + 1/order is diagonally dominant, so it is non-singular with a + # condition number of 2, and its determinant is 2 at every order rather + # than overflowing. An arange matrix is rank 2, which would make solve + # and det meaningless. + self.a = ( + self.np.eye(order, dtype=dt) + + self.np.ones((order, order), dtype=dt) / order + ) + self.b = self.np.ones(order, dtype=dt) + # norm is the cheapest of these, so it warms up the device without + # paying for a second factorization. + self.sync(self.np.linalg.norm(self.a)) + + def time_det(self, executor, order, dtype): + self.sync(self.np.linalg.det(self.a)) + + def time_norm(self, executor, order, dtype): + self.sync(self.np.linalg.norm(self.a)) + + def time_solve(self, executor, order, dtype): + self.sync(self.np.linalg.solve(self.a, self.b)) + + def time_svd(self, executor, order, dtype): + self.sync(self.np.linalg.svd(self.a)) diff --git a/benchmarks/benchmarks/bench_random.py b/benchmarks/benchmarks/bench_random.py index 29ac0ee5e82d..e0b0be542065 100644 --- a/benchmarks/benchmarks/bench_random.py +++ b/benchmarks/benchmarks/bench_random.py @@ -45,6 +45,8 @@ class Sample: def setup(self, executor, size): self.executor = _EXECUTORS[executor] self.sync = make_synchronizer(executor) + # Warm up, so the first timed call does not pay device setup. + self.sync(self.executor.random.rand(size)) def time_rand(self, executor, size): np = self.executor diff --git a/benchmarks/benchmarks/dpbench/README.md b/benchmarks/benchmarks/dpbench/README.md index ca9afef6717f..b61b83366d88 100644 --- a/benchmarks/benchmarks/dpbench/README.md +++ b/benchmarks/benchmarks/dpbench/README.md @@ -28,7 +28,7 @@ config from `dpbench/configs/bench_info/` map onto one module here: ### Intended differences -1. `black_scholes` calls `dpnp.scipy.special.erf`, where dpnp now keeps `erf`. +1. `black_scholes` calls `dpnp.scipy.special.erf`; dpBench calls `dpnp.erf`. 2. Every kernel ends with `dpnp.synchronize_array_data()`. ASV times the `time_*` method directly, so the kernel has to block or only host-side dispatch is measured. diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py index c3dfed4f9e57..df87af288ea7 100644 --- a/benchmarks/benchmarks/dpbench/_dpbench_runner.py +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -89,12 +89,8 @@ def float_dtype(precision): return build_types_dict(precision)["float"] -def preset_fits(workload, preset, device, precision="double"): - """Whether ``preset``'s estimated peak footprint fits ``device``'s memory. - - ``precision`` is the *widest* precision benchmarked, so the verdict is the - same for every precision parameter. - """ +def preset_fits(workload, preset, device, precision): + """Whether ``preset``'s estimated peak footprint fits ``device``'s memory.""" # The cheapest preset always runs, so that an undersized device fails loudly # on allocation rather than reporting nothing. if preset == presets_by_size(workload)[0]: @@ -202,17 +198,25 @@ def relative_error(ref, val): def validate(expected, actual, rel_error=1e-05): """Check that ``actual`` matches ``expected`` closely enough. - Mirrors ``dpbench.infrastructure.benchmark_validation.validate``: a - mismatch is tolerated only while the relative error stays below - ``rel_error``. Raises :exc:`ValueError` naming the offending argument - instead of returning a bool, so a wrong result fails the benchmark rather - than being silently timed. + A mismatch is tolerated only while the relative error stays below + ``rel_error`` and is finite. Raises :exc:`ValueError` naming the offending + argument instead of returning a bool, so a wrong result fails the benchmark + rather than being silently timed. """ for name, ref in expected.items(): val = actual[name] - if numpy.allclose(ref, val): + # equal_nan, so that a NaN the reference also produces counts as + # agreement rather than as the non-finite error rejected below. + if numpy.allclose(ref, val, equal_nan=True): continue + error = relative_error(ref, val) + if not numpy.isfinite(error): + raise ValueError( + f"Validation failed for {name!r}: results differ and the " + "relative error is not finite, so NaN or infinity is present " + "in one of them." + ) if error >= rel_error: raise ValueError( f"Validation failed for {name!r}: relative error {error:.3e} " @@ -278,8 +282,7 @@ def validate(self): def _reference_outputs(self): """Run the NumPy reference on freshly initialized host data.""" - # A fresh initialization is required: the kernel writes into its output - # arrays, so ``self._host_data`` no longer holds their initial values. + # setup() does not retain the host data, so re-initialize it here. host_data = initialize_host_data( self.workload, self.preset, self.precision ) diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 000000000000..2f6903b0c91f --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,7 @@ +# Benchmark-only dependencies, installed into the benchmarking environment by +# CI. asv itself, dpnp and its own dependencies are installed separately. +# +# scipy: scipy.special.erf is used by the NumPy reference that the +# black_scholes benchmark validates its dpnp results against. It is not a dpnp +# runtime dependency, so it has to be requested here. +scipy