From 5b08d5001b2244572c0b284136b51594200c793b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 3 Nov 2021 12:08:43 -0500 Subject: [PATCH 001/229] set the last byte in allocated char array to zero. (#650) --- dpctl-capi/helper/include/dpctl_string_utils.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dpctl-capi/helper/include/dpctl_string_utils.hpp b/dpctl-capi/helper/include/dpctl_string_utils.hpp index 3b1d0ae73c..aa1a8a9535 100644 --- a/dpctl-capi/helper/include/dpctl_string_utils.hpp +++ b/dpctl-capi/helper/include/dpctl_string_utils.hpp @@ -50,6 +50,11 @@ cstring_from_string(const std::string &str) #else std::strncpy(cstr, str.c_str(), cstr_len); #endif + // Added to resolve CheckMarx's false positive. + // NB: This is redundant because str.c_str() is guaranteed + // to be null-terminated and the copy function is asked to + // copy enough characters to include that null-character. + cstr[cstr_len - 1] = '\0'; } catch (std::bad_alloc const &ba) { // \todo log error std::cerr << ba.what() << '\n'; From 3d5d886314b7365e19b62fb7e1595e6701387f00 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 3 Nov 2021 12:09:48 -0500 Subject: [PATCH 002/229] Memory properties (#647) * Add `.sycl_queue` property to MemoryUSM* classes per #640 * Change tests to use `.sycl_queue` property --- dpctl/memory/_memory.pyx | 9 +++++++++ dpctl/tests/test_sycl_usm.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 6a98cf9857..93f69ce548 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -295,6 +295,15 @@ cdef class _Memory: def __get__(self): return self.queue.get_sycl_device() + property sycl_queue: + """ + :class:`dpctl.SyclQueue` with :class:`dpctl.SyclContext` the + USM allocation is bound to and :class:`dpctl.SyclDevice` it was + allocated on. + """ + def __get__(self): + return self.queue + def __repr__(self): return ( "" diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index da19c37668..cb1bfe30d4 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -59,6 +59,8 @@ def test_memory_create(memory_ctor): assert len(mobj) == nbytes assert mobj.size == nbytes assert mobj._context == queue.sycl_context + assert mobj._queue == queue + assert mobj.sycl_queue == queue assert type(repr(mobj)) is str assert type(bytes(mobj)) is bytes assert sys.getsizeof(mobj) > nbytes @@ -495,7 +497,7 @@ def test_with_constructor(memory_ctor): shape=(64,), strides=(1,), offset=0, - syclobj=buf._queue._get_capsule(), + syclobj=buf.sycl_queue._get_capsule(), ) check_view(v) # Use context capsule From be5c0d3fb5d69188b5253524c82856c7302b4750 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 3 Nov 2021 15:57:25 -0500 Subject: [PATCH 003/229] Fixes #652 (#654) --- docs/docfiles/dpctl_pyapi.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docfiles/dpctl_pyapi.rst b/docs/docfiles/dpctl_pyapi.rst index 7c60676d09..dc0dc0fac9 100644 --- a/docs/docfiles/dpctl_pyapi.rst +++ b/docs/docfiles/dpctl_pyapi.rst @@ -28,11 +28,11 @@ Classes .. toctree:: :maxdepth: 1 - dpctl.SyclContext : A Python class representing cl::sycl::context - dpctl.SyclDevice : A Python class representing cl::sycl::device - dpctl.SyclEvent : A Python class representing cl::sycl::event - dpctl.SyclPlatform : A Python class representing cl::sycl::event - dpctl.SyclQueue : A Python class representing cl::sycl::event + dpctl.SyclContext : A Python class representing `sycl::context` + dpctl.SyclDevice : A Python class representing `sycl::device` + dpctl.SyclEvent : A Python class representing `sycl::event` + dpctl.SyclPlatform : A Python class representing `sycl::platform` + dpctl.SyclQueue : A Python class representing `sycl::queue` Enumerations ------------ From 9b04a9ea6ca912b129ddc7194e4061c47c465355 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 8 Nov 2021 11:00:15 -0600 Subject: [PATCH 004/229] Fix/gh 649 transpose (#653) * Fixes #649 ``` In [1]: import dpctl.tensor as dpt, itertools In [2]: a = dpt.usm_ndarray((2,3)) In [3]: for i,j in itertools.product(range(2), range(3)): a[i, j] = i*3 + j In [4]: dpt.to_numpy(a)[1:].T Out[4]: array([[3.], [4.], [5.]]) In [5]: dpt.to_numpy(a[1:].T) Out[5]: array([[3.], [4.], [5.]]) ``` * Fixed issues with `.real` and `.imag` of usm_ndarray These were similar to those reported in #649 * Added tests for T, real, imag methods of the object inspired by #649 --- dpctl/tensor/_usmarray.pyx | 50 +++++++++++++++++++++++----- dpctl/tests/test_usm_ndarray_ctor.py | 22 ++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 65884951de..57c5225a3e 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -1014,14 +1014,30 @@ cdef usm_ndarray _real_view(usm_ndarray ary): """ View into real parts of a complex type array """ - cdef usm_ndarray r = ary._clone() + cdef int r_typenum_ = -1 + cdef usm_ndarray r = None + cdef Py_ssize_t offset_elems = 0 + if (ary.typenum_ == UAR_CFLOAT): - r.typenum_ = UAR_FLOAT + r_typenum_ = UAR_FLOAT elif (ary.typenum_ == UAR_CDOUBLE): - r.typenum_ = UAR_DOUBLE + r_typenum_ = UAR_DOUBLE else: raise InternalUSMArrayError( "_real_view call on array of non-complex type.") + + offset_elems = ary.get_offset() * 2 + r = usm_ndarray.__new__( + usm_ndarray, + _make_int_tuple(ary.nd_, ary.shape_), + dtype=_make_typestr(r_typenum_), + strides=tuple(2 * si for si in ary.strides), + buffer=ary.base_, + offset=offset_elems, + order=('C' if (ary.flags_ & USM_ARRAY_C_CONTIGUOUS) else 'F') + ) + r.flags_ = ary.flags_ + r.array_namespace_ = ary.array_namespace_ return r @@ -1029,16 +1045,31 @@ cdef usm_ndarray _imag_view(usm_ndarray ary): """ View into imaginary parts of a complex type array """ - cdef usm_ndarray r = ary._clone() + cdef int r_typenum_ = -1 + cdef usm_ndarray r = None + cdef Py_ssize_t offset_elems = 0 + if (ary.typenum_ == UAR_CFLOAT): - r.typenum_ = UAR_FLOAT + r_typenum_ = UAR_FLOAT elif (ary.typenum_ == UAR_CDOUBLE): - r.typenum_ = UAR_DOUBLE + r_typenum_ = UAR_DOUBLE else: raise InternalUSMArrayError( - "_real_view call on array of non-complex type.") + "_imag_view call on array of non-complex type.") + # displace pointer to imaginary part - r.data_ = r.data_ + type_bytesize(r.typenum_) + offset_elems = 2 * ary.get_offset() + 1 + r = usm_ndarray.__new__( + usm_ndarray, + _make_int_tuple(ary.nd_, ary.shape_), + dtype=_make_typestr(r_typenum_), + strides=tuple(2 * si for si in ary.strides), + buffer=ary.base_, + offset=offset_elems, + order=('C' if (ary.flags_ & USM_ARRAY_C_CONTIGUOUS) else 'F') + ) + r.flags_ = ary.flags_ + r.array_namespace_ = ary.array_namespace_ return r @@ -1054,7 +1085,8 @@ cdef usm_ndarray _transpose(usm_ndarray ary): _make_reversed_int_tuple(ary.nd_, ary.strides_) if (ary.strides_) else None), buffer=ary.base_, - order=('F' if (ary.flags_ & USM_ARRAY_C_CONTIGUOUS) else 'C') + order=('F' if (ary.flags_ & USM_ARRAY_C_CONTIGUOUS) else 'C'), + offset=ary.get_offset() ) r.flags_ |= (ary.flags_ & USM_ARRAY_WRITEABLE) return r diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index fc78d66bfe..b5fab57566 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -841,3 +841,25 @@ def test_reshape(): dpt.reshape(Z, Z.shape, order="invalid") W = dpt.reshape(Z, (-1,), order="C") assert W.shape == (Z.size,) + + +def test_transpose(): + n, m = 2, 3 + X = dpt.usm_ndarray((n, m), "f4") + Xnp = np.arange(n * m, dtype="f4").reshape((n, m)) + X[:] = Xnp + assert np.array_equal(dpt.to_numpy(X.T), Xnp.T) + assert np.array_equal(dpt.to_numpy(X[1:].T), Xnp[1:].T) + + +def test_real_imag_views(): + n, m = 2, 3 + X = dpt.usm_ndarray((n, m), "c8") + Xnp_r = np.arange(n * m, dtype="f4").reshape((n, m)) + Xnp_i = np.arange(n * m, 2 * n * m, dtype="f4").reshape((n, m)) + Xnp = Xnp_r + 1j * Xnp_i + X[:] = Xnp + assert np.array_equal(dpt.to_numpy(X.real), Xnp.real) + assert np.array_equal(dpt.to_numpy(X.imag), Xnp.imag) + assert np.array_equal(dpt.to_numpy(X[1:].real), Xnp[1:].real) + assert np.array_equal(dpt.to_numpy(X[1:].imag), Xnp[1:].imag) From b7e21bc6941cf9cbb78d3d98190c3dde4e999f22 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 8 Nov 2021 11:07:17 -0600 Subject: [PATCH 005/229] Fixed pip install instruction (#660) Added extra dash for extra-index-url Co-authored-by: samaid --- docs/docfiles/user_guides/QuickStart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index bc3df723dd..f19af4d527 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -56,7 +56,7 @@ Dpctl can also be istalled from Pypi. .. code-block:: bash - python -m pip install --index-url https://pypi.anaconda.org/intel/simple -extra-index-url https://pypi.org/simple dpctl + python -m pip install --index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple dpctl .. note:: From 873d85b896241cd4971b81d7339e77e0d384c3e6 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 8 Nov 2021 15:22:38 -0600 Subject: [PATCH 006/229] Pin coverage at 3.2.0 to work around TheKevJames/coveralls-python#333 (#664) https://github.com/TheKevJames/coveralls-python/issues/333 --- .github/workflows/generate-coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index feb7e71015..776c1316fb 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -93,7 +93,7 @@ jobs: shell: bash -l {0} run: | sudo gem install coveralls-lcov - pip install coveralls + pip install coveralls==3.2.0 - name: Upload coverage data to coveralls.io run: | From e939fe41b77c038834aae979e2542715b69c680a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 9 Nov 2021 10:25:02 -0600 Subject: [PATCH 007/229] Update README.md (#663) * Improve the introductory statement in the README * Update installation steps and remove build steps from README Co-authored-by: samaid Co-authored-by: Diptorup Deb --- README.md | 198 +++++++++++++++++++++--------------------------------- 1 file changed, 76 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 22e170adc4..8f80dda7d2 100644 --- a/README.md +++ b/README.md @@ -2,157 +2,111 @@ [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpctl/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpctl?branch=master) -About dpctl -=========== - -oneAPI logo - -`dpctl` (data parallel control) is a lightweight [Python package](https://intelpython.github.io/dpctl) exposing a -subset of the Intel(R) oneAPI DPC++ [runtime classes](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes) -that is distributed as part of [Intel(R) Distribution for Python*](https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/distribution-for-python.html) and -is included in Intel(R) [oneAPI](https://oneapi.io) [Base ToolKit](https://software.intel.com/content/www/us/en/develop/tools/oneapi/base-toolkit.html). -`dpctl` lets Python users query SYCL platforms, discover and represent SYCL devices, and construct SYCL queues to control data-parallel code execution on [Intel(R) XPUs](https://www.intel.com/content/www/us/en/newsroom/news/xpu-vision-oneapi-server-gpu.html) from Python. - -`dpctl` features classes representing [SYCL unified shared memory](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#sec:usm) -allocations as well as higher-level objects such as [`dpctl.tensor.usm_ndarray`](https://intelpython.github.io/dpctl/latest/docfiles/dpctl.tensor_api.html#module-dpctl.tensor) on top of USM allocations. - -`dpctl` assists authors of Python native extensions written in C, -Cython, or pybind11 to use its `dpctl.SyclQueue` object to indicate the offload -target as well as objects in `dpctl.memory` and `dpctl.tensor` submodules to -represent USM allocations that are accessible from within data-parallel code executed -on the target queue. - -`dpctl.tensor` submodule provides an array container representing an array in a -strided layout on top of a USM allocation. The submodule provides an array-API -conforming oneAPI DPC++ powered library to manipulate the array container. - -Requirements -============ -- Install Conda -- Install Intel oneAPI - - Set environment variable `ONEAPI_ROOT` - - Windows: `C:\Program Files (x86)\Intel\oneAPI\` - - Linux: `/opt/intel/oneapi` -- Install OpenCL HD graphics drivers - -Build and Install Conda Package -================================== -1. Create and activate conda build environment -```bash -conda create -n build-env conda-build -conda activate build-env -``` -2. Set environment variable `ONEAPI_ROOT` and build conda package -```bash -export ONEAPI_ROOT=/opt/intel/oneapi -conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel -``` -On Windows to cope with [long file names](https://github.com/IntelPython/dpctl/issues/15) -use `croot` with short folder path: -```cmd -set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" -conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" -``` - -:warning: **You could face issues with conda-build=3.20**: Use conda-build=3.18! +About +===== -3. Install conda package -```bash -conda install dpctl -``` +oneAPI logo + +Data Parallel Control (`dpctl`) is a Python library that allows a user +to *control* the execution placement of a [compute +kernel](https://en.wikipedia.org/wiki/Compute_kernel) on an +[XPU](https://www.intel.com/content/www/us/en/newsroom/news/xpu-vision-oneapi-server-gpu.html). +The compute kernel can be either a code written by the user, *e.g.*, +using `numba-dppy`, or a code that is part of a library like oneMKL. The `dpctl` +library is built upon the [SYCL +standard](https://www.khronos.org/sycl/) and implements Python +bindings for a subset of the standard [runtime +classes](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes) +that allow users to query platforms, discover and represent devices +and sub-devices, and construct contexts and queues. In addition, +`dpctl` features classes for [SYCL Unified Shared Memory +(USM)](https://link.springer.com/chapter/10.1007/978-1-4842-5574-2_6) +management and implements a tensor [array +API](https://data-apis.org/array-api/latest/). + +The library also assists authors of Python native extensions written +in C, Cython, or pybind11 to access `dpctl` objects representing SYCL +devices, queues, memory, and tensors. + +`Dpctl` is the core part of a larger family of [data-parallel Python +libraries and +tools](https://www.intel.com/content/www/us/en/developer/tools/oneapi/distribution-for-python.html) +to program XPUs. The library is available via +[conda](https://anaconda.org/intel/dpctl) and +[pip](https://pypi.org/project/dpctl/). It is included in the [Intel(R) +Distribution for +Python*](https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/distribution-for-python.html) +(IDP). + +Installing +========== + +From Intel oneAPI +----------------- + +`dpctl` is packaged as part of the quarterly Intel oneAPI releases. To +get the library from the latest oneAPI release please follow the +instructions from Intel's [oneAPI installation +guide](https://www.intel.com/content/www/us/en/developer/articles/guide/installation-guide-for-oneapi-toolkits.html). +Note that you will need to install the Intel BaseKit toolkit to get +IDP and `dpctl`. + +From Conda +---------- + +`dpctl` package is available on the Intel channel on Annaconda +cloud. You an use the following to install `dpctl` from there: -Build and Install with setuptools -================================= -dpctl relies on DPC++ runtime. With Intel oneAPI installed you should activate it. -`setup.py` requires environment variable `ONEAPI_ROOT` and following packages -installed: -- `cython` -- `numpy` -- `cmake` - for building C API -- `ninja` - only on Windows - -You need DPC++ to build dpctl. If you want to build using the DPC++ in a -oneAPI distribution, activate DPC++ compiler as follows: ```bash -export ONEAPI_ROOT=/opt/intel/oneapi -source ${ONEAPI_ROOT}/compiler/latest/env/vars.sh +conda install dpctl -c intel ``` -For install: -```cmd -python setup.py install -``` - -For development: -```cmd -python setup.py develop -``` +From PyPi +--------- -It is also possible to build dpctl using [DPC++ toolchain](https://github.com/intel/llvm/blob/sycl/sycl/doc/GetStartedGuide.md) instead of oneAPI DPC++. Instead of activating the oneAPI environment, indicate the toolchain installation prefix with `--sycl-compiler-prefix` option, e.g. +`dpctl` is also available from PyPi and can be installed using: -```cmd -python setup.py develop --sycl-compiler-prefix=${DPCPP_ROOT}/llvm/build -``` - -Please use `python setup.py develop --help` for more details. - -Install Wheel Package from Pypi -================================== -1. Install dpctl -```cmd -python -m pip install --index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple dpctl +```bash +pip3 install dpctl ``` -Note: dpctl wheel package is placed on Pypi, but some of its dependencies (like Intel numpy) are in Anaconda Cloud. -That is why install command requires additional intel Pypi channel from Anaconda Cloud. -2. Set path to Performance Libraries in case of using venv or system Python: -On Linux: -```cmd -export LD_LIBRARY_PATH=/lib -``` -On Windows: -```cmd -set PATH=\bin;\Library\bin;%PATH% -``` +Installing the bleeding edge +------------------------ -Using dpctl -=========== -dpctl relies on DPC++ runtime. With Intel oneAPI installed you could activate it. +If you want to try out the current master, you can install it from our +development channel on Anaconda cloud: -On Windows: -```cmd -call "%ONEAPI_ROOT%\compiler\latest\env\vars.bat" -``` -On Linux: ```bash -source ${ONEAPI_ROOT}/compiler/latest/env/vars.sh +conda install dpctl -c dppy\label\dev ``` -When dpctl is installed via conda package -then it uses DPC++ runtime from `dpcpp_cpp_rt` package -and it is not necessary to activate oneAPI DPC++ compiler environment. +Building +======== -`dpcpp_cpp_rt` package is provided by oneAPI `conda_channel`. +Please refer our [getting started user +guide](https://intelpython.github.io/dpctl) for more information on +setting up a development environment and building `dpctl` from source. -Examples -======== +Running Examples +================ See examples in folder `examples`. Run python examples: + ```bash for script in `ls examples/python/`; do echo "executing ${script}"; python examples/python/${script}; done ``` Examples of building Cython extensions with DPC++ compiler, that interoperate -with dpctl can be found in folder `cython`. +with `dpctl` can be found in folder `cython`. Each example in `cython` folder can be built using `CC=clang CXX=dpcpp python setup.py build_ext --inplace`. Please refer to `run.py` script in respective folders to execute extensions. -Tests -===== -See tests in folder `dpctl/tests`. +Running Tests +============= +Tests are located in folder `dpctl/tests`. Run tests: ```bash From 0cd7b350b3ef574f0fdfa9355d8bbe34cc52e78c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 9 Nov 2021 14:29:36 -0600 Subject: [PATCH 008/229] Replaced uses of clang compiler with icx executable (#665) --- README.md | 2 +- docs/docfiles/user_guides/QuickStart.rst | 4 ++-- dpctl-capi/cmake/modules/FindIntelSycl.cmake | 2 +- dpctl-capi/dbg_build.bat | 2 +- dpctl-capi/dbg_build.sh | 2 +- examples/cython/sycl_buffer/README.md | 2 +- examples/cython/usm_memory/README.md | 2 +- scripts/build_backend.py | 4 ++-- scripts/build_for_develop.bat | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8f80dda7d2..ffe86c24ee 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ Examples of building Cython extensions with DPC++ compiler, that interoperate with `dpctl` can be found in folder `cython`. Each example in `cython` folder can be built using -`CC=clang CXX=dpcpp python setup.py build_ext --inplace`. +`CC=icx CXX=dpcpp python setup.py build_ext --inplace`. Please refer to `run.py` script in respective folders to execute extensions. Running Tests diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index f19af4d527..dcfede3bba 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -208,7 +208,7 @@ After setting up dpctl you can try out the Python examples as follows: The dpctl repository also provides a set of examples of building Cython extensions with DPC++ compiler, that interoperate with dpctl. These examples are located under *examples/cython*. Each example in the folder can be built using -``CC=clang CXX=dpcpp python setup.py build_ext --inplace``. Please refer to +``CC=icx CXX=dpcpp python setup.py build_ext --inplace``. Please refer to ``run.py`` script in respective folders to execute the Cython extension examples. @@ -252,7 +252,7 @@ library. -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \ -DDPCPP_INSTALL_DIR=${DPCPP_ROOT} \ - -DCMAKE_C_COMPILER:PATH=${DPCPP_ROOT}/bin/clang \ + -DCMAKE_C_COMPILER:PATH=${DPCPP_ROOT}/bin/icx \ -DCMAKE_CXX_COMPILER:PATH=${DPCPP_ROOT}/bin/dpcpp \ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON \ -DDPCTL_BUILD_CAPI_TESTS=ON \ diff --git a/dpctl-capi/cmake/modules/FindIntelSycl.cmake b/dpctl-capi/cmake/modules/FindIntelSycl.cmake index dfc093de44..639e5c4328 100644 --- a/dpctl-capi/cmake/modules/FindIntelSycl.cmake +++ b/dpctl-capi/cmake/modules/FindIntelSycl.cmake @@ -63,7 +63,7 @@ else() endif() # We will extract the version information from the compiler -set(clangxx_cmd "${IntelSycl_ROOT}/bin/clang++") +set(clangxx_cmd "${CMAKE_CXX_COMPILER}") set(clangxx_arg "--version") # Check if dpcpp is available diff --git a/dpctl-capi/dbg_build.bat b/dpctl-capi/dbg_build.bat index 23e8a97097..8149088ed9 100644 --- a/dpctl-capi/dbg_build.bat +++ b/dpctl-capi/dbg_build.bat @@ -26,7 +26,7 @@ cmake -G Ninja ^ "-DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX%" ^ "-DCMAKE_PREFIX_PATH=%INSTALL_PREFIX%" ^ "-DDPCPP_INSTALL_DIR=%DPCPP_HOME%" ^ - "-DCMAKE_C_COMPILER:PATH=%DPCPP_HOME%\bin\clang-cl.exe" ^ + "-DCMAKE_C_COMPILER:PATH=%DPCPP_HOME%\bin\icx.exe" ^ "-DCMAKE_CXX_COMPILER:PATH=%DPCPP_HOME%\bin\dpcpp.exe" ^ "-DCMAKE_LINKER:PATH=%DPCPP_HOME%\bin\lld-link.exe" ^ "-DDPCTL_BUILD_CAPI_TESTS=ON" ^ diff --git a/dpctl-capi/dbg_build.sh b/dpctl-capi/dbg_build.sh index 958a4f8ee3..5b91001240 100755 --- a/dpctl-capi/dbg_build.sh +++ b/dpctl-capi/dbg_build.sh @@ -9,7 +9,7 @@ rm -rf ${INSTALL_PREFIX} cmake \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER=clang \ + -DCMAKE_C_COMPILER=icx \ -DCMAKE_CXX_COMPILER=dpcpp \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \ diff --git a/examples/cython/sycl_buffer/README.md b/examples/cython/sycl_buffer/README.md index 870303bcec..f02b58d1c3 100644 --- a/examples/cython/sycl_buffer/README.md +++ b/examples/cython/sycl_buffer/README.md @@ -14,7 +14,7 @@ Example illustrates compiling SYCL extension, linking to oneMKL. ``` # make sure oneAPI is activated, $ONEAPI_ROOT must be set -CC=clang CXX=dpcpp python setup.py build_ext --inplace +CC=icx CXX=dpcpp python setup.py build_ext --inplace ``` diff --git a/examples/cython/usm_memory/README.md b/examples/cython/usm_memory/README.md index 0f17a77dba..300e4464a8 100644 --- a/examples/cython/usm_memory/README.md +++ b/examples/cython/usm_memory/README.md @@ -8,7 +8,7 @@ Make sure oneAPI is activated. Environment variable `$ONEAPI_ROOT` must be set. ``` -$ CC=clang CXX=dpcpp LD_SHARED="dpcpp -shared" \ +$ CC=icx CXX=dpcpp LD_SHARED="dpcpp -shared" \ CXXFLAGS=-fno-sycl-early-optimizations python setup.py build_ext --inplace ``` diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 4f26951d5b..6b705a4341 100644 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -86,7 +86,7 @@ def build_backend( "-DDPCTL_DPCPP_HOME_DIR=" + DPCPP_ROOT, "-DDPCTL_DPCPP_FROM_ONEAPI=ON", "-DCMAKE_C_COMPILER:PATH=" - + os.path.join(DPCPP_ROOT, "bin", "clang"), + + os.path.join(DPCPP_ROOT, "bin", "icx"), "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "dpcpp"), ] @@ -164,7 +164,7 @@ def build_backend( "-DDPCTL_DPCPP_HOME_DIR=" + DPCPP_ROOT, "-DDPCTL_DPCPP_FROM_ONEAPI=ON", "-DCMAKE_C_COMPILER:PATH=" - + os.path.join(DPCPP_ROOT, "bin", "clang-cl.exe"), + + os.path.join(DPCPP_ROOT, "bin", "icx.exe"), "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "dpcpp.exe"), ] diff --git a/scripts/build_for_develop.bat b/scripts/build_for_develop.bat index 17c4fe1e70..0273a9d62a 100644 --- a/scripts/build_for_develop.bat +++ b/scripts/build_for_develop.bat @@ -31,7 +31,7 @@ cmake -G Ninja ^ "-DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX%" ^ "-DCMAKE_PREFIX_PATH=%INSTALL_PREFIX%" ^ "-DDPCPP_ROOT=%DPCPP_ROOT%" ^ - "-DCMAKE_C_COMPILER:PATH=%DPCPP_ROOT%\bin\clang-cl.exe" ^ + "-DCMAKE_C_COMPILER:PATH=%DPCPP_ROOT%\bin\icx.exe" ^ "-DCMAKE_CXX_COMPILER:PATH=%DPCPP_ROOT%\bin\dpcpp.exe" ^ "-DBUILD_CAPI_TESTS=%_BUILD_CAPI_TEST%" ^ "%cd%\..\dpctl-capi" @@ -57,7 +57,7 @@ xcopy dpctl-capi\include dpctl\include /E /Y REM required by _sycl_core(dpctl) set "DPCTL_SYCL_INTERFACE_LIBDIR=dpctl" set "DPCTL_SYCL_INTERFACE_INCLDIR=dpctl\include" -set "CC=clang-cl.exe" +set "CC=icx.exe" set "CXX=dpcpp.exe" python setup.py clean --all From f292953f56d6f26fa8f96158fdd4e6d00b78c491 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 9 Nov 2021 16:09:35 -0600 Subject: [PATCH 009/229] Update dpctl_pyapi.rst (#661) Rearranged the order of classes and functions to make the documentation flow more logical. Provided actual links to the SYCL Reference Manual describing Programs and Kernels. Co-authored-by: samaid --- docs/docfiles/dpctl_pyapi.rst | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/docfiles/dpctl_pyapi.rst b/docs/docfiles/dpctl_pyapi.rst index dc0dc0fac9..4b9ec30fef 100644 --- a/docs/docfiles/dpctl_pyapi.rst +++ b/docs/docfiles/dpctl_pyapi.rst @@ -14,13 +14,14 @@ Sub-modules :mod:`dpctl.memory` USM allocators and deallocators and classes that implement Python's `buffer protocol`_. + :mod:`dpctl.tensor` + Implementation of different types of tensor classes that use USM memory. :mod:`dpctl.program` - Experimental wrappers for SYCL 1.2 ``program`` and ``kernel`` classes. + Experimental wrappers for SYCL 1.2 `Program` + and `Kernel` classes. The module is going to be refactored in the future to support SYCL - 2020's ``kernel_bundle`` feature and the wrapper for the ``program`` + 2020's ``sycl::kernel_bundle`` feature and the wrapper for the ``sycl::program`` class is going to be removed. - :mod:`dpctl.tensor` - Implementation of different types of tensor classes that use USM memory. Classes ------- @@ -28,11 +29,11 @@ Classes .. toctree:: :maxdepth: 1 - dpctl.SyclContext : A Python class representing `sycl::context` - dpctl.SyclDevice : A Python class representing `sycl::device` - dpctl.SyclEvent : A Python class representing `sycl::event` - dpctl.SyclPlatform : A Python class representing `sycl::platform` - dpctl.SyclQueue : A Python class representing `sycl::queue` + `dpctl.SyclPlatform` : A Python class representing `SYCL Platforms` + `dpctl.SyclDevice` : A Python class representing `SYCL Devices` + `dpctl.SyclQueue` : A Python class representing `SYCL Queues` + `dpctl.SyclContext` : A Python class representing `SYCL Contexts` + `dpctl.SyclEvent` : A Python class representing `SYCL Events` Enumerations ------------ @@ -53,17 +54,17 @@ Exceptions Device Selection Functions -------------------------- -.. autofunction:: get_devices -.. autofunction:: select_accelerator_device -.. autofunction:: select_cpu_device -.. autofunction:: select_default_device -.. autofunction:: select_gpu_device -.. autofunction:: select_host_device .. autofunction:: get_num_devices +.. autofunction:: get_devices +.. autofunction:: has_host_device .. autofunction:: has_cpu_devices .. autofunction:: has_gpu_devices .. autofunction:: has_accelerator_devices -.. autofunction:: has_host_device +.. autofunction:: select_default_device +.. autofunction:: select_cpu_device +.. autofunction:: select_gpu_device +.. autofunction:: select_host_device +.. autofunction:: select_accelerator_device Queue Management Functions -------------------------- From 309aa1f53dde7cd17c96f194773d31d5da2029f4 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 9 Nov 2021 18:30:05 -0600 Subject: [PATCH 010/229] Fix the hyperlinks to dpctl Python Classes API docs. (#667) --- docs/docfiles/dpctl_pyapi.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/docfiles/dpctl_pyapi.rst b/docs/docfiles/dpctl_pyapi.rst index 4b9ec30fef..c7c66c63d8 100644 --- a/docs/docfiles/dpctl_pyapi.rst +++ b/docs/docfiles/dpctl_pyapi.rst @@ -26,14 +26,19 @@ Sub-modules Classes ------- -.. toctree:: - :maxdepth: 1 - - `dpctl.SyclPlatform` : A Python class representing `SYCL Platforms` - `dpctl.SyclDevice` : A Python class representing `SYCL Devices` - `dpctl.SyclQueue` : A Python class representing `SYCL Queues` - `dpctl.SyclContext` : A Python class representing `SYCL Contexts` - `dpctl.SyclEvent` : A Python class representing `SYCL Events` +.. list-table:: + :widths: 25,50 + + * - :ref:`SyclPlatform_api` + - A Python class representing a `SYCL platform `_. + * - :ref:`SyclDevice_api` + - A Python class representing a `SYCL device `_. + * - :ref:`SyclQueue_api` + - A Python class representing a `SYCL queue `_. + * - :ref:`SyclContext_api` + - A Python class representing a `SYCL context `_. + * - :ref:`SyclEvent_api` + - A Python class representing a `SYCL event `_. Enumerations ------------ From 5c3fed5f5600d337d3d4505eb9bb01878b9efd79 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 10 Nov 2021 11:37:06 -0600 Subject: [PATCH 011/229] Rename dpctl capi (#666) * Rename dpctl-capi to libsyclinterface. * Update CONTRIBUTING.md to use new name for dpctl-capi. * Update folder name in cpp_style_checks workflow. * Update Changelog. * Change to libsyclinterface in docs. --- .github/workflows/cpp_style_checks.yml | 2 +- CHANGELOG.md | 5 +++++ CONTRIBUTING.md | 12 ++++++------ docs/.gitignore | 2 +- docs/CMakeLists.txt | 4 ++-- docs/Doxyfile.in | 10 +++++----- docs/doxyrest-config.lua.in | 2 +- {dpctl-capi => libsyclinterface}/.gitignore | 0 {dpctl-capi => libsyclinterface}/CMakeLists.txt | 0 .../cmake/modules/FindIntelSycl.cmake | 0 .../cmake/modules/FindLLVMCov.cmake | 0 .../cmake/modules/FindLLVMProfdata.cmake | 0 .../cmake/modules/FindLcov.cmake | 0 .../cmake/modules/GetLevelZeroHeaders.cmake | 0 .../cmake/modules/GetProjectVersion.cmake | 0 {dpctl-capi => libsyclinterface}/dbg_build.bat | 0 {dpctl-capi => libsyclinterface}/dbg_build.sh | 0 .../dbg_build_custom.sh | 0 .../helper/include/dpctl_async_error_handler.h | 0 .../helper/include/dpctl_dynamic_lib_helper.h | 0 .../helper/include/dpctl_string_utils.hpp | 0 .../helper/include/dpctl_utils_helper.h | 0 .../helper/include/dpctl_vector_macros.h | 0 .../helper/source/dpctl_async_error_handler.cpp | 0 .../helper/source/dpctl_utils_helper.cpp | 0 .../include/Config/.gitignore | 0 .../include/Config/dpctl_config.h.in | 0 .../include/Support/CBindingWrapping.h | 0 .../include/Support/DllExport.h | 0 .../include/Support/ExternC.h | 0 .../include/Support/MemOwnershipAttrs.h | 0 .../include/dpctl_data_types.h | 0 .../include/dpctl_error_handler_type.h | 0 .../include/dpctl_service.h | 0 .../include/dpctl_sycl_context_interface.h | 0 .../include/dpctl_sycl_device_interface.h | 0 .../include/dpctl_sycl_device_manager.h | 0 .../include/dpctl_sycl_device_selector_interface.h | 0 .../include/dpctl_sycl_enum_types.h | 0 .../include/dpctl_sycl_event_interface.h | 0 .../include/dpctl_sycl_kernel_interface.h | 0 .../include/dpctl_sycl_platform_interface.h | 0 .../include/dpctl_sycl_platform_manager.h | 0 .../include/dpctl_sycl_program_interface.h | 0 .../include/dpctl_sycl_queue_interface.h | 0 .../include/dpctl_sycl_queue_manager.h | 0 .../include/dpctl_sycl_types.h | 0 .../include/dpctl_sycl_usm_interface.h | 0 .../include/dpctl_utils.h | 0 .../include/dpctl_vector.h | 0 .../source/dpctl_service.cpp | 0 .../source/dpctl_sycl_context_interface.cpp | 0 .../source/dpctl_sycl_device_interface.cpp | 0 .../source/dpctl_sycl_device_manager.cpp | 0 .../source/dpctl_sycl_device_selector_interface.cpp | 0 .../source/dpctl_sycl_event_interface.cpp | 0 .../source/dpctl_sycl_kernel_interface.cpp | 0 .../source/dpctl_sycl_platform_interface.cpp | 0 .../source/dpctl_sycl_platform_manager.cpp | 0 .../source/dpctl_sycl_program_interface.cpp | 0 .../source/dpctl_sycl_queue_interface.cpp | 0 .../source/dpctl_sycl_queue_manager.cpp | 0 .../source/dpctl_sycl_usm_interface.cpp | 0 .../source/dpctl_utils.cpp | 0 .../source/dpctl_vector_templ.cpp | 0 .../tests/CMakeLists.txt | 0 .../tests/dpcpp_kernels.cpp | 0 .../tests/dpcpp_kernels.hpp | 0 .../tests/multi_kernel.spv | Bin .../tests/test_main.cpp | 0 .../tests/test_service.cpp | 0 .../tests/test_sycl_context_interface.cpp | 0 .../tests/test_sycl_device_aspects.cpp | 0 .../tests/test_sycl_device_interface.cpp | 0 .../tests/test_sycl_device_invalid_filters.cpp | 0 .../tests/test_sycl_device_manager.cpp | 0 .../tests/test_sycl_device_selector_interface.cpp | 0 .../tests/test_sycl_device_subdevices.cpp | 0 .../tests/test_sycl_event_interface.cpp | 0 .../tests/test_sycl_kernel_interface.cpp | 0 .../tests/test_sycl_platform_interface.cpp | 0 .../tests/test_sycl_platform_invalid_filters.cpp | 0 .../tests/test_sycl_program_interface.cpp | 0 .../tests/test_sycl_queue_interface.cpp | 0 .../tests/test_sycl_queue_manager.cpp | 0 .../tests/test_sycl_queue_submit.cpp | 0 .../tests/test_sycl_usm_interface.cpp | 0 scripts/build_backend.py | 4 ++-- 88 files changed, 23 insertions(+), 18 deletions(-) rename {dpctl-capi => libsyclinterface}/.gitignore (100%) rename {dpctl-capi => libsyclinterface}/CMakeLists.txt (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/FindIntelSycl.cmake (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/FindLLVMCov.cmake (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/FindLLVMProfdata.cmake (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/FindLcov.cmake (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/GetLevelZeroHeaders.cmake (100%) rename {dpctl-capi => libsyclinterface}/cmake/modules/GetProjectVersion.cmake (100%) rename {dpctl-capi => libsyclinterface}/dbg_build.bat (100%) rename {dpctl-capi => libsyclinterface}/dbg_build.sh (100%) rename {dpctl-capi => libsyclinterface}/dbg_build_custom.sh (100%) rename {dpctl-capi => libsyclinterface}/helper/include/dpctl_async_error_handler.h (100%) rename {dpctl-capi => libsyclinterface}/helper/include/dpctl_dynamic_lib_helper.h (100%) rename {dpctl-capi => libsyclinterface}/helper/include/dpctl_string_utils.hpp (100%) rename {dpctl-capi => libsyclinterface}/helper/include/dpctl_utils_helper.h (100%) rename {dpctl-capi => libsyclinterface}/helper/include/dpctl_vector_macros.h (100%) rename {dpctl-capi => libsyclinterface}/helper/source/dpctl_async_error_handler.cpp (100%) rename {dpctl-capi => libsyclinterface}/helper/source/dpctl_utils_helper.cpp (100%) rename {dpctl-capi => libsyclinterface}/include/Config/.gitignore (100%) rename {dpctl-capi => libsyclinterface}/include/Config/dpctl_config.h.in (100%) rename {dpctl-capi => libsyclinterface}/include/Support/CBindingWrapping.h (100%) rename {dpctl-capi => libsyclinterface}/include/Support/DllExport.h (100%) rename {dpctl-capi => libsyclinterface}/include/Support/ExternC.h (100%) rename {dpctl-capi => libsyclinterface}/include/Support/MemOwnershipAttrs.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_data_types.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_error_handler_type.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_service.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_context_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_device_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_device_manager.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_device_selector_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_enum_types.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_event_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_kernel_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_platform_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_platform_manager.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_program_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_queue_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_queue_manager.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_types.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_sycl_usm_interface.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_utils.h (100%) rename {dpctl-capi => libsyclinterface}/include/dpctl_vector.h (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_service.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_context_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_device_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_device_manager.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_device_selector_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_event_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_kernel_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_platform_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_platform_manager.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_program_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_queue_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_queue_manager.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_sycl_usm_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_utils.cpp (100%) rename {dpctl-capi => libsyclinterface}/source/dpctl_vector_templ.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/CMakeLists.txt (100%) rename {dpctl-capi => libsyclinterface}/tests/dpcpp_kernels.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/dpcpp_kernels.hpp (100%) rename {dpctl-capi => libsyclinterface}/tests/multi_kernel.spv (100%) rename {dpctl-capi => libsyclinterface}/tests/test_main.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_service.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_context_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_aspects.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_invalid_filters.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_manager.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_selector_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_device_subdevices.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_event_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_kernel_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_platform_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_platform_invalid_filters.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_program_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_queue_interface.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_queue_manager.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_queue_submit.cpp (100%) rename {dpctl-capi => libsyclinterface}/tests/test_sycl_usm_interface.cpp (100%) diff --git a/.github/workflows/cpp_style_checks.yml b/.github/workflows/cpp_style_checks.yml index b5a7def26f..89ba110ccf 100644 --- a/.github/workflows/cpp_style_checks.yml +++ b/.github/workflows/cpp_style_checks.yml @@ -19,4 +19,4 @@ jobs: uses: jidicula/clang-format-action@v3.1.0 with: clang-format-version: '11' - check-path: 'dpctl-capi' + check-path: 'libsyclinterface' diff --git a/CHANGELOG.md b/CHANGELOG.md index f155ed5380..765fba02f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Next Release] TBD + +### Changed +- dpctl-capi is now renamed to `libsyclinterface` (#666). + ## [0.11.0] - 11/01/2021 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cae87e86e..b1bf79975e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,12 +15,12 @@ Run before each commit: ```bash clang-format -style=file -i \ - dpctl-capi/include/*.h \ - dpctl-capi/include/Support/*.h \ - dpctl-capi/source/*.cpp \ - dpctl-capi/tests/*.cpp \ - dpctl-capi/helper/include/*.h \ - dpctl-capi/helper/source/*.cpp + libsyclinterface/include/*.h \ + libsyclinterface/include/Support/*.h \ + libsyclinterface/source/*.cpp \ + libsyclinterface/tests/*.cpp \ + libsyclinterface/helper/include/*.h \ + libsyclinterface/helper/source/*.cpp ``` > **_NOTE:_** A much simpler option is to use `pre-commit` and the diff --git a/docs/.gitignore b/docs/.gitignore index c7b38c7ac8..27d532e775 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,6 +1,6 @@ docs generated_docs -docfiles/dpctl-capi +docfiles/libsyclinterface api build conf.py diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index bba66d788a..8e34d6b886 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -29,7 +29,7 @@ function(_setup_doxygen) if(DPCTL_ENABLE_DOXYGEN_HTML) set(GENERATE_HTML "YES") endif() - set(DOXYGEN_INPUT_DIR ../dpctl-capi/include) + set(DOXYGEN_INPUT_DIR ../libsyclinterface/include) set(DOXYGEN_OUTPUT_DIR ${DOC_OUTPUT_DIR}/doxygen) set(DOXYGEN_INDEX_FILE ${DOXYGEN_OUTPUT_DIR}/xml/index.xml) set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) @@ -61,7 +61,7 @@ function(_setup_doxygen) endfunction() function(_setup_doxyrest) - set(DOXYREST_OUTPUT_DIR_NAME docfiles/dpctl-capi) + set(DOXYREST_OUTPUT_DIR_NAME docfiles/libsyclinterface) set(DOXYREST_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DOXYREST_OUTPUT_DIR_NAME} PARENT_SCOPE diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index a1055b7256..934262a3e6 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -917,11 +917,11 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = ../dpctl-capi/include/Support -EXCLUDE += ../dpctl-capi/include/Config -EXCLUDE += ../dpctl-capi/include/dpctl_vector.h -EXCLUDE += ../dpctl-capi/include/dpctl_data_types.h -EXCLUDE += ../dpctl-capi/include/dpctl_utils.h +EXCLUDE = ../libsyclinterface/include/Support +EXCLUDE += ../libsyclinterface/include/Config +EXCLUDE += ../libsyclinterface/include/dpctl_vector.h +EXCLUDE += ../libsyclinterface/include/dpctl_data_types.h +EXCLUDE += ../libsyclinterface/include/dpctl_utils.h # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/docs/doxyrest-config.lua.in b/docs/doxyrest-config.lua.in index 857260ddeb..42de9e7797 100644 --- a/docs/doxyrest-config.lua.in +++ b/docs/doxyrest-config.lua.in @@ -82,7 +82,7 @@ EXTRA_PAGE_LIST = {} --! is not set (otherwise, the title of intro file will be used). --! -INDEX_TITLE = "dpctl C API" +INDEX_TITLE = "libsyclinterface" --! --! File with project introduction (reStructuredText). When non-nil, this file diff --git a/dpctl-capi/.gitignore b/libsyclinterface/.gitignore similarity index 100% rename from dpctl-capi/.gitignore rename to libsyclinterface/.gitignore diff --git a/dpctl-capi/CMakeLists.txt b/libsyclinterface/CMakeLists.txt similarity index 100% rename from dpctl-capi/CMakeLists.txt rename to libsyclinterface/CMakeLists.txt diff --git a/dpctl-capi/cmake/modules/FindIntelSycl.cmake b/libsyclinterface/cmake/modules/FindIntelSycl.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindIntelSycl.cmake rename to libsyclinterface/cmake/modules/FindIntelSycl.cmake diff --git a/dpctl-capi/cmake/modules/FindLLVMCov.cmake b/libsyclinterface/cmake/modules/FindLLVMCov.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLLVMCov.cmake rename to libsyclinterface/cmake/modules/FindLLVMCov.cmake diff --git a/dpctl-capi/cmake/modules/FindLLVMProfdata.cmake b/libsyclinterface/cmake/modules/FindLLVMProfdata.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLLVMProfdata.cmake rename to libsyclinterface/cmake/modules/FindLLVMProfdata.cmake diff --git a/dpctl-capi/cmake/modules/FindLcov.cmake b/libsyclinterface/cmake/modules/FindLcov.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLcov.cmake rename to libsyclinterface/cmake/modules/FindLcov.cmake diff --git a/dpctl-capi/cmake/modules/GetLevelZeroHeaders.cmake b/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake similarity index 100% rename from dpctl-capi/cmake/modules/GetLevelZeroHeaders.cmake rename to libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake diff --git a/dpctl-capi/cmake/modules/GetProjectVersion.cmake b/libsyclinterface/cmake/modules/GetProjectVersion.cmake similarity index 100% rename from dpctl-capi/cmake/modules/GetProjectVersion.cmake rename to libsyclinterface/cmake/modules/GetProjectVersion.cmake diff --git a/dpctl-capi/dbg_build.bat b/libsyclinterface/dbg_build.bat similarity index 100% rename from dpctl-capi/dbg_build.bat rename to libsyclinterface/dbg_build.bat diff --git a/dpctl-capi/dbg_build.sh b/libsyclinterface/dbg_build.sh similarity index 100% rename from dpctl-capi/dbg_build.sh rename to libsyclinterface/dbg_build.sh diff --git a/dpctl-capi/dbg_build_custom.sh b/libsyclinterface/dbg_build_custom.sh similarity index 100% rename from dpctl-capi/dbg_build_custom.sh rename to libsyclinterface/dbg_build_custom.sh diff --git a/dpctl-capi/helper/include/dpctl_async_error_handler.h b/libsyclinterface/helper/include/dpctl_async_error_handler.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_async_error_handler.h rename to libsyclinterface/helper/include/dpctl_async_error_handler.h diff --git a/dpctl-capi/helper/include/dpctl_dynamic_lib_helper.h b/libsyclinterface/helper/include/dpctl_dynamic_lib_helper.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_dynamic_lib_helper.h rename to libsyclinterface/helper/include/dpctl_dynamic_lib_helper.h diff --git a/dpctl-capi/helper/include/dpctl_string_utils.hpp b/libsyclinterface/helper/include/dpctl_string_utils.hpp similarity index 100% rename from dpctl-capi/helper/include/dpctl_string_utils.hpp rename to libsyclinterface/helper/include/dpctl_string_utils.hpp diff --git a/dpctl-capi/helper/include/dpctl_utils_helper.h b/libsyclinterface/helper/include/dpctl_utils_helper.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_utils_helper.h rename to libsyclinterface/helper/include/dpctl_utils_helper.h diff --git a/dpctl-capi/helper/include/dpctl_vector_macros.h b/libsyclinterface/helper/include/dpctl_vector_macros.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_vector_macros.h rename to libsyclinterface/helper/include/dpctl_vector_macros.h diff --git a/dpctl-capi/helper/source/dpctl_async_error_handler.cpp b/libsyclinterface/helper/source/dpctl_async_error_handler.cpp similarity index 100% rename from dpctl-capi/helper/source/dpctl_async_error_handler.cpp rename to libsyclinterface/helper/source/dpctl_async_error_handler.cpp diff --git a/dpctl-capi/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp similarity index 100% rename from dpctl-capi/helper/source/dpctl_utils_helper.cpp rename to libsyclinterface/helper/source/dpctl_utils_helper.cpp diff --git a/dpctl-capi/include/Config/.gitignore b/libsyclinterface/include/Config/.gitignore similarity index 100% rename from dpctl-capi/include/Config/.gitignore rename to libsyclinterface/include/Config/.gitignore diff --git a/dpctl-capi/include/Config/dpctl_config.h.in b/libsyclinterface/include/Config/dpctl_config.h.in similarity index 100% rename from dpctl-capi/include/Config/dpctl_config.h.in rename to libsyclinterface/include/Config/dpctl_config.h.in diff --git a/dpctl-capi/include/Support/CBindingWrapping.h b/libsyclinterface/include/Support/CBindingWrapping.h similarity index 100% rename from dpctl-capi/include/Support/CBindingWrapping.h rename to libsyclinterface/include/Support/CBindingWrapping.h diff --git a/dpctl-capi/include/Support/DllExport.h b/libsyclinterface/include/Support/DllExport.h similarity index 100% rename from dpctl-capi/include/Support/DllExport.h rename to libsyclinterface/include/Support/DllExport.h diff --git a/dpctl-capi/include/Support/ExternC.h b/libsyclinterface/include/Support/ExternC.h similarity index 100% rename from dpctl-capi/include/Support/ExternC.h rename to libsyclinterface/include/Support/ExternC.h diff --git a/dpctl-capi/include/Support/MemOwnershipAttrs.h b/libsyclinterface/include/Support/MemOwnershipAttrs.h similarity index 100% rename from dpctl-capi/include/Support/MemOwnershipAttrs.h rename to libsyclinterface/include/Support/MemOwnershipAttrs.h diff --git a/dpctl-capi/include/dpctl_data_types.h b/libsyclinterface/include/dpctl_data_types.h similarity index 100% rename from dpctl-capi/include/dpctl_data_types.h rename to libsyclinterface/include/dpctl_data_types.h diff --git a/dpctl-capi/include/dpctl_error_handler_type.h b/libsyclinterface/include/dpctl_error_handler_type.h similarity index 100% rename from dpctl-capi/include/dpctl_error_handler_type.h rename to libsyclinterface/include/dpctl_error_handler_type.h diff --git a/dpctl-capi/include/dpctl_service.h b/libsyclinterface/include/dpctl_service.h similarity index 100% rename from dpctl-capi/include/dpctl_service.h rename to libsyclinterface/include/dpctl_service.h diff --git a/dpctl-capi/include/dpctl_sycl_context_interface.h b/libsyclinterface/include/dpctl_sycl_context_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_context_interface.h rename to libsyclinterface/include/dpctl_sycl_context_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_device_interface.h b/libsyclinterface/include/dpctl_sycl_device_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_interface.h rename to libsyclinterface/include/dpctl_sycl_device_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_device_manager.h b/libsyclinterface/include/dpctl_sycl_device_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_manager.h rename to libsyclinterface/include/dpctl_sycl_device_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_device_selector_interface.h b/libsyclinterface/include/dpctl_sycl_device_selector_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_selector_interface.h rename to libsyclinterface/include/dpctl_sycl_device_selector_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_enum_types.h b/libsyclinterface/include/dpctl_sycl_enum_types.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_enum_types.h rename to libsyclinterface/include/dpctl_sycl_enum_types.h diff --git a/dpctl-capi/include/dpctl_sycl_event_interface.h b/libsyclinterface/include/dpctl_sycl_event_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_event_interface.h rename to libsyclinterface/include/dpctl_sycl_event_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_kernel_interface.h b/libsyclinterface/include/dpctl_sycl_kernel_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_kernel_interface.h rename to libsyclinterface/include/dpctl_sycl_kernel_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_platform_interface.h b/libsyclinterface/include/dpctl_sycl_platform_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_platform_interface.h rename to libsyclinterface/include/dpctl_sycl_platform_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_platform_manager.h b/libsyclinterface/include/dpctl_sycl_platform_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_platform_manager.h rename to libsyclinterface/include/dpctl_sycl_platform_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_program_interface.h b/libsyclinterface/include/dpctl_sycl_program_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_program_interface.h rename to libsyclinterface/include/dpctl_sycl_program_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_queue_interface.h b/libsyclinterface/include/dpctl_sycl_queue_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_queue_interface.h rename to libsyclinterface/include/dpctl_sycl_queue_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_queue_manager.h b/libsyclinterface/include/dpctl_sycl_queue_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_queue_manager.h rename to libsyclinterface/include/dpctl_sycl_queue_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_types.h b/libsyclinterface/include/dpctl_sycl_types.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_types.h rename to libsyclinterface/include/dpctl_sycl_types.h diff --git a/dpctl-capi/include/dpctl_sycl_usm_interface.h b/libsyclinterface/include/dpctl_sycl_usm_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_usm_interface.h rename to libsyclinterface/include/dpctl_sycl_usm_interface.h diff --git a/dpctl-capi/include/dpctl_utils.h b/libsyclinterface/include/dpctl_utils.h similarity index 100% rename from dpctl-capi/include/dpctl_utils.h rename to libsyclinterface/include/dpctl_utils.h diff --git a/dpctl-capi/include/dpctl_vector.h b/libsyclinterface/include/dpctl_vector.h similarity index 100% rename from dpctl-capi/include/dpctl_vector.h rename to libsyclinterface/include/dpctl_vector.h diff --git a/dpctl-capi/source/dpctl_service.cpp b/libsyclinterface/source/dpctl_service.cpp similarity index 100% rename from dpctl-capi/source/dpctl_service.cpp rename to libsyclinterface/source/dpctl_service.cpp diff --git a/dpctl-capi/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_context_interface.cpp rename to libsyclinterface/source/dpctl_sycl_context_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_device_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_device_interface.cpp rename to libsyclinterface/source/dpctl_sycl_device_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_device_manager.cpp rename to libsyclinterface/source/dpctl_sycl_device_manager.cpp diff --git a/dpctl-capi/source/dpctl_sycl_device_selector_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_device_selector_interface.cpp rename to libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_event_interface.cpp b/libsyclinterface/source/dpctl_sycl_event_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_event_interface.cpp rename to libsyclinterface/source/dpctl_sycl_event_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_kernel_interface.cpp b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_kernel_interface.cpp rename to libsyclinterface/source/dpctl_sycl_kernel_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_platform_interface.cpp b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_platform_interface.cpp rename to libsyclinterface/source/dpctl_sycl_platform_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_platform_manager.cpp b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_platform_manager.cpp rename to libsyclinterface/source/dpctl_sycl_platform_manager.cpp diff --git a/dpctl-capi/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_program_interface.cpp rename to libsyclinterface/source/dpctl_sycl_program_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_queue_interface.cpp rename to libsyclinterface/source/dpctl_sycl_queue_interface.cpp diff --git a/dpctl-capi/source/dpctl_sycl_queue_manager.cpp b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_queue_manager.cpp rename to libsyclinterface/source/dpctl_sycl_queue_manager.cpp diff --git a/dpctl-capi/source/dpctl_sycl_usm_interface.cpp b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp similarity index 100% rename from dpctl-capi/source/dpctl_sycl_usm_interface.cpp rename to libsyclinterface/source/dpctl_sycl_usm_interface.cpp diff --git a/dpctl-capi/source/dpctl_utils.cpp b/libsyclinterface/source/dpctl_utils.cpp similarity index 100% rename from dpctl-capi/source/dpctl_utils.cpp rename to libsyclinterface/source/dpctl_utils.cpp diff --git a/dpctl-capi/source/dpctl_vector_templ.cpp b/libsyclinterface/source/dpctl_vector_templ.cpp similarity index 100% rename from dpctl-capi/source/dpctl_vector_templ.cpp rename to libsyclinterface/source/dpctl_vector_templ.cpp diff --git a/dpctl-capi/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt similarity index 100% rename from dpctl-capi/tests/CMakeLists.txt rename to libsyclinterface/tests/CMakeLists.txt diff --git a/dpctl-capi/tests/dpcpp_kernels.cpp b/libsyclinterface/tests/dpcpp_kernels.cpp similarity index 100% rename from dpctl-capi/tests/dpcpp_kernels.cpp rename to libsyclinterface/tests/dpcpp_kernels.cpp diff --git a/dpctl-capi/tests/dpcpp_kernels.hpp b/libsyclinterface/tests/dpcpp_kernels.hpp similarity index 100% rename from dpctl-capi/tests/dpcpp_kernels.hpp rename to libsyclinterface/tests/dpcpp_kernels.hpp diff --git a/dpctl-capi/tests/multi_kernel.spv b/libsyclinterface/tests/multi_kernel.spv similarity index 100% rename from dpctl-capi/tests/multi_kernel.spv rename to libsyclinterface/tests/multi_kernel.spv diff --git a/dpctl-capi/tests/test_main.cpp b/libsyclinterface/tests/test_main.cpp similarity index 100% rename from dpctl-capi/tests/test_main.cpp rename to libsyclinterface/tests/test_main.cpp diff --git a/dpctl-capi/tests/test_service.cpp b/libsyclinterface/tests/test_service.cpp similarity index 100% rename from dpctl-capi/tests/test_service.cpp rename to libsyclinterface/tests/test_service.cpp diff --git a/dpctl-capi/tests/test_sycl_context_interface.cpp b/libsyclinterface/tests/test_sycl_context_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_context_interface.cpp rename to libsyclinterface/tests/test_sycl_context_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_device_aspects.cpp b/libsyclinterface/tests/test_sycl_device_aspects.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_aspects.cpp rename to libsyclinterface/tests/test_sycl_device_aspects.cpp diff --git a/dpctl-capi/tests/test_sycl_device_interface.cpp b/libsyclinterface/tests/test_sycl_device_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_interface.cpp rename to libsyclinterface/tests/test_sycl_device_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_device_invalid_filters.cpp b/libsyclinterface/tests/test_sycl_device_invalid_filters.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_invalid_filters.cpp rename to libsyclinterface/tests/test_sycl_device_invalid_filters.cpp diff --git a/dpctl-capi/tests/test_sycl_device_manager.cpp b/libsyclinterface/tests/test_sycl_device_manager.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_manager.cpp rename to libsyclinterface/tests/test_sycl_device_manager.cpp diff --git a/dpctl-capi/tests/test_sycl_device_selector_interface.cpp b/libsyclinterface/tests/test_sycl_device_selector_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_selector_interface.cpp rename to libsyclinterface/tests/test_sycl_device_selector_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_device_subdevices.cpp b/libsyclinterface/tests/test_sycl_device_subdevices.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_subdevices.cpp rename to libsyclinterface/tests/test_sycl_device_subdevices.cpp diff --git a/dpctl-capi/tests/test_sycl_event_interface.cpp b/libsyclinterface/tests/test_sycl_event_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_event_interface.cpp rename to libsyclinterface/tests/test_sycl_event_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_kernel_interface.cpp b/libsyclinterface/tests/test_sycl_kernel_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_kernel_interface.cpp rename to libsyclinterface/tests/test_sycl_kernel_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_platform_interface.cpp b/libsyclinterface/tests/test_sycl_platform_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_platform_interface.cpp rename to libsyclinterface/tests/test_sycl_platform_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_platform_invalid_filters.cpp b/libsyclinterface/tests/test_sycl_platform_invalid_filters.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_platform_invalid_filters.cpp rename to libsyclinterface/tests/test_sycl_platform_invalid_filters.cpp diff --git a/dpctl-capi/tests/test_sycl_program_interface.cpp b/libsyclinterface/tests/test_sycl_program_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_program_interface.cpp rename to libsyclinterface/tests/test_sycl_program_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_queue_interface.cpp b/libsyclinterface/tests/test_sycl_queue_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_queue_interface.cpp rename to libsyclinterface/tests/test_sycl_queue_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_queue_manager.cpp b/libsyclinterface/tests/test_sycl_queue_manager.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_queue_manager.cpp rename to libsyclinterface/tests/test_sycl_queue_manager.cpp diff --git a/dpctl-capi/tests/test_sycl_queue_submit.cpp b/libsyclinterface/tests/test_sycl_queue_submit.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_queue_submit.cpp rename to libsyclinterface/tests/test_sycl_queue_submit.cpp diff --git a/dpctl-capi/tests/test_sycl_usm_interface.cpp b/libsyclinterface/tests/test_sycl_usm_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_usm_interface.cpp rename to libsyclinterface/tests/test_sycl_usm_interface.cpp diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 6b705a4341..0ee4e8799a 100644 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -76,7 +76,7 @@ def build_backend( if os.path.exists(INSTALL_PREFIX): shutil.rmtree(INSTALL_PREFIX) - backends = os.path.join(dpctl_dir, "dpctl-capi") + backends = os.path.join(dpctl_dir, "libsyclinterface") ENABLE_LO_PROGRAM_CREATION = "ON" if l0_support else "OFF" @@ -213,7 +213,7 @@ def build_backend( shutil.rmtree(include_dir) shutil.copytree( - os.path.join(dpctl_dir, "dpctl-capi", "include"), include_dir + os.path.join(dpctl_dir, "libsyclinterface", "include"), include_dir ) From 9d264fdc244dd4a68bdbce23ca3b31529d99996c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 10 Nov 2021 11:53:22 -0600 Subject: [PATCH 012/229] Update _sycl_queue_manager.pyx (#659) Improved device_context(arg) API reference documentation Co-authored-by: samaid --- dpctl/_sycl_queue_manager.pyx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dpctl/_sycl_queue_manager.pyx b/dpctl/_sycl_queue_manager.pyx index 53d5058d04..c2b854da12 100644 --- a/dpctl/_sycl_queue_manager.pyx +++ b/dpctl/_sycl_queue_manager.pyx @@ -213,7 +213,8 @@ cpdef get_current_backend(): @contextmanager def device_context(arg): """ - Yields a SYCL queue corresponding to the input filter string. + Yields a SYCL queue corresponding to the input queue object, device object, + or device filter selector string. This context manager "activates", *i.e.*, sets as the currently usable queue, the SYCL queue defined by the argument `arg`. @@ -223,25 +224,25 @@ def device_context(arg): usable queue on exiting the context manager. Args: - - queue_str (str) : A string corresponding to the DPC++ filter selector. + arg : A :class:`dpctl.SyclQueue` object, or a :class:`dpctl.SyclDevice` + object, or a filter selector string. Yields: - :class:`.SyclQueue`: A SYCL queue corresponding to the specified - filter string. + :class:`dpctl.SyclQueue`: A SYCL queue corresponding to the specified + input device, queue, or filter string. Raises: SyclQueueCreationError: If the SYCL queue creation failed. :Example: - To create a scope within which the Level Zero GPU number 0 is active, - a programmer needs to do the following. + The following example sets current queue targeting specific device + indicated with filter selector string in the scope of `with` block: .. code-block:: python import dpctl with dpctl.device_context("level0:gpu:0"): - pass + do_something_on_gpu0() """ ctxt = None From 9df69c94f724c39cce348781b00d8edbb81da4d4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 10 Nov 2021 17:34:34 -0600 Subject: [PATCH 013/229] Try to publish generated_docs from PR sources and comment on the PR (#673) * Try to publish generated_docs from PR sources and comment on the PR * Added logic for tearing down pulls/prNum after PR is closed --- .github/workflows/generate-docs.yml | 68 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index be23c32363..986da24322 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -3,6 +3,8 @@ on: push: branches: - master + pull_request: + types: [opened, synchronize, reopened, closed] jobs: build-and-deploy: @@ -14,6 +16,7 @@ jobs: with: access_token: ${{ github.token }} - name: Add Intel repository + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB @@ -21,23 +24,29 @@ jobs: sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" sudo apt-get update - name: Install Intel OneAPI + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install intel-oneapi-dpcpp-cpp-compiler - name: Install Lua + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install liblua5.2-dev - name: Install Doxygen + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install doxygen - name: Install CMake + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install cmake - name: Setup Python + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} uses: actions/setup-python@v2 with: python-version: '3.8' architecture: x64 - name: Install sphinx dependencies + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | pip install numpy cython setuptools sphinx sphinx_rtd_theme pydot graphviz @@ -46,12 +55,14 @@ jobs: with: fetch-depth: 0 - name: Build dpctl + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | source /opt/intel/oneapi/setvars.sh python setup.py develop python -c "import dpctl; print(dpctl.__version__)" - name: Build docs + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | # Ensure that SYCL libraries are on LD_LIBRARY_PATH @@ -69,6 +80,7 @@ jobs: git clean -dfx cd .. - name: Publish docs + if: ${{ github.ref == 'refs/heads/master' }} shell: bash -l {0} run: | git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git @@ -80,6 +92,60 @@ jobs: mv ~/docs/* . git add . git config --global user.name 'github-actions[doc-deploy-bot]' - git config --gloabl user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' git commit -m "Latest docs." git push tokened_docs gh-pages + - name: Publish pull-request docs + if: ${{ github.event.pull_request && github.event.action != 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + shell: bash -l {0} + run: | + git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git + git fetch tokened_docs + git checkout --track tokened_docs/gh-pages + echo `pwd` + [ -d pulls/${PR_NUM} ] && git rm -rf pulls/${PR_NUM} + mkdir -p pulls/${PR_NUM} + cd pulls/${PR_NUM} + mv ~/docs/* . + git add . + git config --global user.name 'github-actions[doc-deploy-bot]' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git commit -m "Docs for pull request ${PR_NUM}" + git push tokened_docs gh-pages + - name: Unpublished pull-request docs + if: ${{ github.event.pull_request && github.event.action == 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + shell: bash -l {0} + run: | + git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git + git fetch tokened_docs + git checkout --track tokened_docs/gh-pages + echo `pwd` + [ -d pulls/${PR_NUM} ] && git rm -rf pulls/${PR_NUM} + git config --global user.name 'github-actions[doc-deploy-bot]' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git commit -m "Removing docs for closed pull request ${PR_NUM}" + git push tokened_docs gh-pages + - name: Comment with URL to published pull-request docs + if: ${{ github.event.pull_request && github.event.action != 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + uses: mshick/add-pr-comment@v1 + with: + message: | + View rendered docs @ https://intelpython.github.io/dpctl/pulls/${{ env.PR_NUM }}/index.html + repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token-user-login: 'github-actions[bot]' + - name: Comment with URL about removal of PR docs + if: ${{ github.event.pull_request && github.event.action == 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + uses: mshick/add-pr-comment@v1 + with: + message: | + Deleted rendered PR docs from intelpython.github.com/dpctl, latest should be updated shortly. :crossed_fingers: + repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token-user-login: 'github-actions[bot]' From c0baab1a26089201fd05e5760c234924d4141ae3 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 10 Nov 2021 21:33:54 -0600 Subject: [PATCH 014/229] Set stype fields in ZeModuleDesc struct (#672) Set structure type to ZE_STRUCTURE_TYPE_MODULE_DESC --- libsyclinterface/source/dpctl_sycl_program_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index 46b32c2de0..fc2f443beb 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -159,6 +159,7 @@ createLevelZeroInterOpProgram(const context &SyclCtx, // Populate the Level Zero module descriptions ze_module_desc_t ZeModuleDesc = {}; + ZeModuleDesc.stype = ZE_STRUCTURE_TYPE_MODULE_DESC; ZeModuleDesc.format = ZE_MODULE_FORMAT_IL_SPIRV; ZeModuleDesc.inputSize = length; ZeModuleDesc.pInputModule = (uint8_t *)IL; From 0b5e0f3601f046675a097fb72864d04edbd9ab40 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 11 Nov 2021 19:48:58 -0600 Subject: [PATCH 015/229] Attempt to fix permission on push issue (#675) --- .github/workflows/generate-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 986da24322..f5b780b0bd 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -54,6 +54,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + persist-credentials: false - name: Build dpctl if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} From 8213b0b623d05f40ccd6de04e5ff75a5224e8232 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Thu, 11 Nov 2021 19:51:00 -0600 Subject: [PATCH 016/229] Update versions for tools used in pre-commit checks. Add a badge. (#674) --- .pre-commit-config.yaml | 8 ++++---- README.md | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebc971590c..54286f0c65 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,17 +8,17 @@ repos: pass_filenames: false args: ["-r", "dpctl", "-lll"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.0.1 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 21.4b2 + rev: 21.10b0 hooks: - id: black exclude: "versioneer.py|dpctl/_version.py" - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort name: isort (python) @@ -29,7 +29,7 @@ repos: name: isort (pyi) types: [pyi] - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.1 + rev: 4.0.1 hooks: - id: flake8 - repo: https://github.com/pocc/pre-commit-hooks diff --git a/README.md b/README.md index ffe86c24ee..6d06fdf92d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpctl/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpctl?branch=master) About From 749be95b3b54b5fd1933b4f910295fd224640d6a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 19 Nov 2021 20:34:23 -0600 Subject: [PATCH 017/229] Update CHANGELOG for 0.11.1 (#680) Co-authored-by: Sergey Pokhodenko --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 765fba02f3..4af5ac2d79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Next Release] TBD +## [Next Release] - TBD -### Changed - dpctl-capi is now renamed to `libsyclinterface` (#666). +## [0.11.1] - 11/10/2021 + +### Changed +- Replaced uses of clang compiler with icx executable (#665) + ## [0.11.0] - 11/01/2021 ### Added From 9d0050c9fcbd5287894bd7df0a50ff63e4189864 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 29 Oct 2021 14:40:17 -0500 Subject: [PATCH 018/229] Add initial implementation of dpctl.tensor.asarray, dpctl.tensor.empty ``` dpctl.tensor.asarray( obj, dtype=None, copy=None, order="C", device=None, usm_type=None, sycl_queue=None ) ``` ``` dpctl.tensor.empty( shape, dtype="f8", order="C", device=None, usm_type=None, sycl_queue=None ) ``` --- dpctl/tensor/__init__.py | 3 + dpctl/tensor/_ctors.py | 377 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 dpctl/tensor/_ctors.py diff --git a/dpctl/tensor/__init__.py b/dpctl/tensor/__init__.py index 32e1e1ce46..6036099c62 100644 --- a/dpctl/tensor/__init__.py +++ b/dpctl/tensor/__init__.py @@ -33,13 +33,16 @@ from dpctl.tensor._copy_utils import copy_from_numpy as from_numpy from dpctl.tensor._copy_utils import copy_to_numpy as asnumpy from dpctl.tensor._copy_utils import copy_to_numpy as to_numpy +from dpctl.tensor._ctors import asarray, empty from dpctl.tensor._reshape import reshape from dpctl.tensor._usmarray import usm_ndarray __all__ = [ "usm_ndarray", + "asarray", "astype", "copy", + "empty", "reshape", "from_numpy", "to_numpy", diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py new file mode 100644 index 0000000000..14e42f65a5 --- /dev/null +++ b/dpctl/tensor/_ctors.py @@ -0,0 +1,377 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +import dpctl +import dpctl.memory as dpm +import dpctl.tensor as dpt +import dpctl.utils + +_empty_tuple = tuple() +_host_set = frozenset([None]) + + +def _array_info_dispatch(obj): + if isinstance(obj, dpt.usm_ndarray): + return obj.shape, obj.dtype, frozenset([obj.sycl_queue]) + elif isinstance(obj, np.ndarray): + return obj.shape, obj.dtype, _host_set + elif isinstance(obj, range): + return (len(obj),), int, _host_set + elif isinstance(obj, float): + return _empty_tuple, float, _host_set + elif isinstance(obj, int): + return _empty_tuple, int, _host_set + elif isinstance(obj, complex): + return _empty_tuple, complex, _host_set + elif isinstance(obj, (list, tuple, range)): + return _array_info_sequence(obj) + else: + raise ValueError(type(obj)) + + +def _array_info_sequence(li): + assert isinstance(li, (list, tuple, range)) + n = len(li) + dim = None + dt = None + device = frozenset() + for el in li: + el_dim, el_dt, el_dev = _array_info_dispatch(el) + if dim is None: + dim = el_dim + dt = np.promote_types(el_dt, el_dt) + device = device.union(el_dev) + elif el_dim == dim: + dt = np.promote_types(dt, el_dt) + device = device.union(el_dev) + else: + raise ValueError( + "Inconsistent dimensions, {} and {}".format(dim, el_dim) + ) + return (n,) + dim, dt, device + + +def _normalize_queue_device(q=None, d=None): + if q is None: + d = dpt._device.Device.create_device(d) + return d.sycl_queue + else: + if not isinstance(q, dpctl.SyclQueue): + raise TypeError(f"Expected dpctl.SyclQueue, got {type(q)}") + if d is None: + return q + d = dpt._device.Device.create_device(d) + qq = dpctl.utils.get_execution_queue( + ( + q, + d.sycl_queue, + ) + ) + if qq is None: + raise TypeError( + "sycl_queue and device keywords can not be both specified" + ) + return qq + + +def _asarray_from_usm_ndarray( + usm_ndary, + dtype=None, + copy=None, + usm_type=None, + sycl_queue=None, + order="K", +): + if not isinstance(usm_ndary, dpt.usm_ndarray): + raise TypeError( + f"Expected dpctl.tensor.usm_ndarray, got {type(usm_ndary)}" + ) + if dtype is None: + dtype = usm_ndary.dtype + if usm_type is None: + usm_type = usm_ndary.usm_type + if sycl_queue is not None: + exec_q = dpctl.utils.get_execution_queue( + [usm_ndary.sycl_queue, sycl_queue] + ) + copy_q = _normalize_queue_device(q=sycl_queue, d=exec_q) + else: + copy_q = usm_ndary.sycl_queue + # Conditions for zero copy: + can_zero_copy = copy is not True + # dtype is unchanged + can_zero_copy = can_zero_copy and dtype == usm_ndary.dtype + # USM allocation type is unchanged + can_zero_copy = can_zero_copy and usm_type == usm_ndary.usm_type + # sycl_queue is unchanged + can_zero_copy = can_zero_copy and copy_q is usm_ndary.sycl_queue + # order is unchanged + c_contig = usm_ndary.flags & 1 + f_contig = usm_ndary.flags & 2 + fc_contig = usm_ndary.flags & 3 + if can_zero_copy: + if order == "C" and c_contig: + pass + elif order == "F" and f_contig: + pass + elif order == "A" and fc_contig: + pass + elif order == "K": + pass + else: + can_zero_copy = False + if copy is False and can_zero_copy is False: + raise ValueError("asarray(..., copy=False) is not possible") + if can_zero_copy: + return usm_ndary + if order == "A": + order = "F" if f_contig and not c_contig else "C" + if order == "K" and fc_contig: + order = "C" if c_contig else "F" + if order == "K": + # new USM allocation + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=dtype, + buffer=usm_type, + order="C", + buffer_ctor_kwargs={"queue": copy_q}, + ) + original_strides = usm_ndary.strides + ind = sorted( + range(usm_ndary.ndim), + key=lambda i: abs(original_strides[i]), + reverse=True, + ) + new_strides = tuple(res.strides[ind[i]] for i in ind) + # reuse previously made USM allocation + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=res.dtype, + buffer=res.usm_data, + strides=new_strides, + ) + else: + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": copy_q}, + ) + # FIXME: call copy_to when implemented + res[(slice(None, None, None),) * res.ndim] = usm_ndary + return res + + +def _asarray_from_numpy_ndarray( + ary, dtype=None, usm_type=None, sycl_queue=None, order="K" +): + if not isinstance(ary, np.ndarray): + raise TypeError(f"Expected numpy.ndarray, got {type(ary)}") + if usm_type is None: + usm_type = "device" + if dtype is None: + dtype = ary.dtype + copy_q = _normalize_queue_device(q=None, d=sycl_queue) + f_contig = ary.flags["F"] + c_contig = ary.flags["C"] + fc_contig = f_contig or c_contig + if order == "A": + order = "F" if f_contig and not c_contig else "C" + if order == "K" and fc_contig: + order = "C" if c_contig else "F" + if order == "K": + # new USM allocation + res = dpt.usm_ndarray( + ary.shape, + dtype=dtype, + buffer=usm_type, + order="C", + buffer_ctor_kwargs={"queue": copy_q}, + ) + original_strides = ary.strides + ind = sorted( + range(ary.ndim), + key=lambda i: abs(original_strides[i]), + reverse=True, + ) + new_strides = tuple(res.strides[ind[i]] for i in ind) + # reuse previously made USM allocation + res = dpt.usm_ndarray( + res.shape, dtype=res.dtype, buffer=res.usm_data, strides=new_strides + ) + else: + res = dpt.usm_ndarray( + ary.shape, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": copy_q}, + ) + # FIXME: call copy_to when implemented + res[(slice(None, None, None),) * res.ndim] = ary + return res + + +def asarray( + obj, + dtype=None, + device=None, + copy=None, + usm_type=None, + sycl_queue=None, + order="K", +): + """Represents object `obj` as usm_ndarray""" + # 1. Check that copy is a valid keyword + if copy not in [None, True, False]: + raise TypeError( + "Recognized copy keyword values should be True, False, or None" + ) + # 2. Check that dtype is None, or a valid dtype + if dtype is not None: + dtype = np.dtype(dtype) + # 3. Validate order + if not isinstance(order, str): + raise TypeError( + f"Expected order keyword to be of type str, got {type(order)}" + ) + if len(order) == 0 or order[0] not in "KkAaCcFf": + raise ValueError( + "Unrecognized order keyword value, expecting 'K', 'A', 'F', or 'C'." + ) + else: + order = order[0].upper() + # 4. Check that usm_type is None, or a valid value + if usm_type is not None: + if isinstance(usm_type, str): + if usm_type not in ["device", "shared", "host"]: + raise ValueError( + f"Unrecognized value of usm_type={usm_type}, " + "expected 'device', 'shared', 'host', or None." + ) + else: + raise TypeError( + f"Expected usm_type to be a str or None, got {type(usm_type)}" + ) + # 5. Normalize device/sycl_queue [keep it None if was None] + if device is not None or sycl_queue is not None: + sycl_queue = _normalize_queue_device(q=sycl_queue, d=device) + + # handle instance(obj, usm_ndarray) + if isinstance(obj, dpt.usm_ndarray): + return _asarray_from_usm_ndarray( + obj, + dtype=dtype, + copy=copy, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif hasattr(obj, "__sycl_usm_array_interface__"): + sua_iface = getattr(obj, "__sycl_usm_array_interface__") + membuf = dpm.as_usm_memory(obj) + ary = dpt.usm_ndarray( + sua_iface["shape"], + dtype=sua_iface["typestr"], + buffer=membuf, + strides=sua_iface.get("strides", None), + ) + return _asarray_from_usm_ndarray( + ary, + dtype=dtype, + copy=copy, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif isinstance(obj, np.ndarray): + if copy is False: + raise ValueError( + "Converting numpy.ndarray to usm_ndarray requires a copy" + ) + return _asarray_from_numpy_ndarray( + obj, + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif hasattr(obj, "__dlpack__"): + raise NotImplementedError( + "Support for __dlpack__ is not yet implemented" + ) + elif isinstance(obj, (list, tuple, range)): + if copy is False: + raise ValueError( + "Converting Python sequence to usm_ndarray requires a copy" + ) + _, dt, devs = _array_info_sequence(obj) + if devs == _host_set: + return _asarray_from_numpy_ndarray( + np.asarray(obj, dt, order=order), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + # for sequences + raise NotImplementedError( + "Converting Python sequences is not implemented" + ) + # obj is a scalar, create 0d array + return _asarray_from_numpy_ndarray( + np.asarray(obj), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order="C", + ) + + +def empty( + sh, dtype="f8", order="C", device=None, usm_type="device", sycl_queue=None +): + """Creates empty usm_ndarray""" + dtype = np.dtype(dtype) + if len(order) == 0 or order[0] not in "CcFf": + raise ValueError( + "Unrecognized order keyword value, expecting 'F' or 'C'." + ) + else: + order = order[0].upper() + if isinstance(usm_type, str): + if usm_type not in ["device", "shared", "host"]: + raise ValueError( + f"Unrecognized value of usm_type={usm_type}, " + "expected 'device', 'shared', or 'host'." + ) + else: + raise TypeError( + f"Expected usm_type to be of type str, got {type(usm_type)}" + ) + sycl_queue = _normalize_queue_device(q=sycl_queue, d=device) + res = dpt.usm_ndarray( + sh, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": sycl_queue}, + ) + return res From e4e7682adbf0094dc5b2a79f92ff0dad1d8f244a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 29 Oct 2021 15:11:25 -0500 Subject: [PATCH 019/229] Adding test file for asarray and empty 1. Check validation of arguments is `asarray` 2. Check that `asarray` works with objects exposing `__sycl_usm_array_interface__`. Adding a test with scalar arguments, tests to cover input validation, tests for copy=False --- dpctl/tests/test_tensor_asarray.py | 192 +++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 dpctl/tests/test_tensor_asarray.py diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py new file mode 100644 index 0000000000..a17a470b5f --- /dev/null +++ b/dpctl/tests/test_tensor_asarray.py @@ -0,0 +1,192 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest + +import dpctl +import dpctl.tensor as dpt + + +@pytest.mark.parametrize( + "src_usm_type, dst_usm_type", + [ + ("device", "shared"), + ("device", "host"), + ("shared", "device"), + ("shared", "host"), + ("host", "device"), + ("host", "shared"), + ], +) +def test_asarray_change_usm_type(src_usm_type, dst_usm_type): + d = dpctl.SyclDevice() + if d.is_host: + pytest.skip( + "Skip test of host device, which only " + "supports host USM allocations" + ) + X = dpt.empty(10, dtype="u1", usm_type=src_usm_type) + Y = dpt.asarray(X, usm_type=dst_usm_type) + assert X.shape == Y.shape + assert X.usm_type == src_usm_type + assert Y.usm_type == dst_usm_type + + with pytest.raises(ValueError): + # zero copy is not possible + dpt.asarray(X, usm_type=dst_usm_type, copy=False) + + Y = dpt.asarray(X, usm_type=dst_usm_type, sycl_queue=X.sycl_queue) + assert X.shape == Y.shape + assert Y.usm_type == dst_usm_type + + Y = dpt.asarray( + X, + usm_type=dst_usm_type, + sycl_queue=X.sycl_queue, + device=d.get_filter_string(), + ) + assert X.shape == Y.shape + assert Y.usm_type == dst_usm_type + + +def test_asarray_from_numpy(): + Xnp = np.arange(10) + Y = dpt.asarray(Xnp, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == (10,) + assert Y.dtype == Xnp.dtype + + +def test_asarray_from_sequence(): + X = [1, 2, 3] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + + X = [(1, 1), (2.0, 2.0 + 1.0j), range(4, 6), np.array([3, 4], dtype="c16")] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.ndim == 2 + + +def test_asarray_from_object_with_suai(): + """Test that asarray can deal with opaque objects implementing SUAI""" + + class Dummy: + def __init__(self, obj, iface): + self.obj = obj + self.__sycl_usm_array_interface__ = iface + + X = dpt.empty((2, 3, 4), dtype="f4") + Y = dpt.asarray(Dummy(X, X.__sycl_usm_array_interface__)) + assert Y.shape == X.shape + assert X.usm_type == Y.usm_type + assert X.dtype == Y.dtype + assert X.sycl_device == Y.sycl_device + + +def test_asarray_input_validation(): + with pytest.raises(TypeError): + # copy keyword is not of right type + dpt.asarray([1], copy="invalid") + with pytest.raises(TypeError): + # order keyword is not valid + dpt.asarray([1], order=1) + with pytest.raises(TypeError): + # dtype is not valid + dpt.asarray([1], dtype="invalid") + with pytest.raises(ValueError): + # unexpected value of order + dpt.asarray([1], order="Z") + with pytest.raises(TypeError): + # usm_type is of wrong type + dpt.asarray([1], usm_type=dict()) + with pytest.raises(ValueError): + # usm_type has wrong value + dpt.asarray([1], usm_type="mistake") + with pytest.raises(TypeError): + # sycl_queue type is not right + dpt.asarray([1], sycl_queue=dpctl.SyclContext()) + with pytest.raises(ValueError): + # sequence is not rectangular + dpt.asarray([[1], 2]) + + +def test_asarray_input_validation2(): + d = dpctl.get_devices() + if len(d) < 2: + pytest.skip("Not enough SYCL devices available") + + d0, d1 = d[:2] + try: + q0 = dpctl.SyclQueue(d0) + except dpctl.SyclQueueCreationError: + pytest.skip(f"SyclQueue could not be created for {d0}") + try: + q1 = dpctl.SyclQueue(d1) + except dpctl.SyclQueueCreationError: + pytest.skip(f"SyclQueue could not be created for {d1}") + with pytest.raises(TypeError): + dpt.asarray([1, 2], sycl_queue=q0, device=q1) + + +def test_asarray_scalars(): + import ctypes + + Y = dpt.asarray(5) + assert Y.dtype == np.dtype(int) + Y = dpt.asarray(5.2) + assert Y.dtype == np.dtype(float) + Y = dpt.asarray(np.float32(2.3)) + assert Y.dtype == np.dtype(np.float32) + Y = dpt.asarray(1.0j) + assert Y.dtype == np.dtype(complex) + Y = dpt.asarray(ctypes.c_int(8)) + assert Y.dtype == np.dtype(ctypes.c_int) + + +def test_asarray_copy_false(): + try: + q = dpctl.SyclQueue() + except dpctl.SyclQueueCreationError: + pytest.skip("Could not create a queue") + X = dpt.from_numpy(np.random.randn(10, 4), usm_type="device", queue=q) + Y1 = dpt.asarray(X, copy=False, order="K") + assert Y1 is X + Y1c = dpt.asarray(X, copy=True, order="K") + assert not (Y1c is X) + Y2 = dpt.asarray(X, copy=False, order="C") + assert Y2 is X + Y3 = dpt.asarray(X, copy=False, order="A") + assert Y3 is X + with pytest.raises(ValueError): + Y1 = dpt.asarray(X, copy=False, order="F") + Xf = dpt.empty( + X.shape, + dtype=X.dtype, + usm_type="device", + sycl_queue=X.sycl_queue, + order="F", + ) + Xf[:] = X + Y4 = dpt.asarray(Xf, copy=False, order="K") + assert Y4 is Xf + Y5 = dpt.asarray(Xf, copy=False, order="F") + assert Y5 is Xf + Y6 = dpt.asarray(Xf, copy=False, order="A") + assert Y6 is Xf + with pytest.raises(ValueError): + dpt.asarray(Xf, copy=False, order="C") From 9c49c5db67f5eeea0d3c66bec44587fcde5e3d51 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 18 Nov 2021 16:05:07 -0600 Subject: [PATCH 020/229] Added docstrings to asarray and empty Change default of asarray's order keyword from "K" to "C" Fixed issue pointed out by Denis (processing of order in empty) Added support for objects with buffer protocol in asarray. --- dpctl/tensor/_ctors.py | 99 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index 14e42f65a5..d0f459d1c5 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -229,6 +229,17 @@ def _asarray_from_numpy_ndarray( return res +def _is_object_with_buffer_protocol(obj): + "Returns True if object support Python buffer protocol" + try: + # use context manager to ensure + # buffer is instantly released + with memoryview(obj): + return True + except TypeError: + return False + + def asarray( obj, dtype=None, @@ -238,7 +249,45 @@ def asarray( sycl_queue=None, order="K", ): - """Represents object `obj` as usm_ndarray""" + """asarray(obj, dtype=None, copy=None, order="K", + device=None, usm_type=None, sycl_queue=None) + + Converts `obj` to :class:`dpctl.tensor.usm_ndarray`. + + Args: + obj: Python object to convert. Can be an instance of `usm_ndarray`, + an object representing SYCL USM allocation and implementing + `__sycl_usm_array_interface__` protocol, an instance + of `numpy.ndarray`, an object supporting Python buffer protocol, + a Python scalar, or a (possibly nested) sequence of Python scalars. + dtype (data type, optional): output array data type. If `dtype` is + `None`, the output array data type is inferred from data types in + `obj`. Default: `None` + copy (`bool`, optional): boolean indicating whether or not to copy the + input. If `True`, always creates a copy. If `False`, need to copy + raises `ValueError`. If `None`, try to reuse existing memory + allocations if possible, but allowed to perform a copy otherwise. + Default: `None`. + order ("C","F","A","K", optional): memory layout of the output array. + Default: "C" + device (optional): array API concept of device where the output array + is created. `device` can be `None`, a oneAPI filter selector string, + an instance of :class:`dpctl.SyclDevice` corresponding to a + non-partitioned SYCL device, an instance of + :class:`dpctl.SyclQueue`, or a `Device` object returnedby + `dpctl.tensor.usm_array.device`. Default: `None`. + usm_type ("device"|"shared"|"host", optional): The type of SYCL USM + allocation for the output array. For `usm_type=None` the allocation + type is inferred from the input if `obj` has USM allocation, or + `"device"` is used instead. Default: `None`. + sycl_queue: (:class:`dpctl.SyclQueue`, optional): The SYCL queue to use + for output array allocation and copying. `sycl_queue` and `device` + are exclusive keywords, i.e. use one or another. If both are + specified, a `TypeError` is raised unless both imply the same + underlying SYCL queue to be used. If both a `None`, the + `dpctl.SyclQueue()` is used for allocation and copying. + Default: `None`. + """ # 1. Check that copy is a valid keyword if copy not in [None, True, False]: raise TypeError( @@ -313,9 +362,17 @@ def asarray( sycl_queue=sycl_queue, order=order, ) - elif hasattr(obj, "__dlpack__"): - raise NotImplementedError( - "Support for __dlpack__ is not yet implemented" + elif _is_object_with_buffer_protocol(obj): + if copy is False: + raise ValueError( + f"Converting {type(obj)} to usm_ndarray requires a copy" + ) + return _asarray_from_numpy_ndarray( + np.array(obj), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, ) elif isinstance(obj, (list, tuple, range)): if copy is False: @@ -335,6 +392,10 @@ def asarray( raise NotImplementedError( "Converting Python sequences is not implemented" ) + if copy is False: + raise ValueError( + f"Converting {type(obj)} to usm_ndarray requires a copy" + ) # obj is a scalar, create 0d array return _asarray_from_numpy_ndarray( np.asarray(obj), @@ -348,9 +409,35 @@ def asarray( def empty( sh, dtype="f8", order="C", device=None, usm_type="device", sycl_queue=None ): - """Creates empty usm_ndarray""" + """dpctl.tensor.empty(shape, dtype="f8", order="C", device=None, + usm_type="device", sycl_queue=None) + + Creates `usm_ndarray` from uninitialized USM allocation. + + Args: + shape (tuple): Dimensions of the array to be created. + dtype (optional): data type of the array. Can be typestring, + a `numpy.dtype` object, `numpy` char string, or a numpy + scalar type. Default: "f8" + order ("C", or F"): memory layout for the array. Default: "C" + device (optional): array API concept of device where the output array + is created. `device` can be `None`, a oneAPI filter selector string, + an instance of :class:`dpctl.SyclDevice` corresponding to a + non-partitioned SYCL device, an instance of + :class:`dpctl.SyclQueue`, or a `Device` object returnedby + `dpctl.tensor.usm_array.device`. Default: `None`. + usm_type ("device"|"shared"|"host", optional): The type of SYCL USM + allocation for the output array. Default: `"device"`. + sycl_queue: (:class:`dpctl.SyclQueue`, optional): The SYCL queue to use + for output array allocation and copying. `sycl_queue` and `device` + are exclusive keywords, i.e. use one or another. If both are + specified, a `TypeError` is raised unless both imply the same + underlying SYCL queue to be used. If both a `None`, the + `dpctl.SyclQueue()` is used for allocation and copying. + Default: `None`. + """ dtype = np.dtype(dtype) - if len(order) == 0 or order[0] not in "CcFf": + if not isinstance(order, str) or len(order) == 0 or order[0] not in "CcFf": raise ValueError( "Unrecognized order keyword value, expecting 'F' or 'C'." ) From 1626363abd5568cddd9e47374f70558ca4a02db5 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 19 Nov 2021 13:53:37 -0600 Subject: [PATCH 021/229] Populate CHANGELOG/added list with functions from feature/asarray PR --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af5ac2d79..c09fea7af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Next Release] - TBD +### Added +- `dpctl.tensor.asarray`, `dpctl.tensor.empty` implemented (#646). + +### Changed - dpctl-capi is now renamed to `libsyclinterface` (#666). ## [0.11.1] - 11/10/2021 From 12d6693647d3817f5ea6eeec4883eaa0bdab28bb Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sat, 20 Nov 2021 08:36:41 -0600 Subject: [PATCH 022/229] renamed keyword in dpctl.tensor.from_numpy, and its alias asnumpy from queue to sycl_queue --- dpctl/tensor/_copy_utils.py | 6 +++--- dpctl/tests/test_tensor_asarray.py | 2 +- dpctl/tests/test_usm_ndarray_ctor.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index 7a7ba918f6..9422210a6d 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -93,12 +93,12 @@ def copy_to_numpy(ary): ) -def copy_from_numpy(np_ary, usm_type="device", queue=None): +def copy_from_numpy(np_ary, usm_type="device", sycl_queue=None): "Copies numpy array `np_ary` into a new usm_ndarray" # This may peform a copy to meet stated requirements Xnp = np.require(np_ary, requirements=["A", "O", "C", "E"]) - if queue: - ctor_kwargs = {"queue": queue} + if sycl_queue: + ctor_kwargs = {"queue": sycl_queue} else: ctor_kwargs = dict() Xusm = dpt.usm_ndarray( diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py index a17a470b5f..c7734b8194 100644 --- a/dpctl/tests/test_tensor_asarray.py +++ b/dpctl/tests/test_tensor_asarray.py @@ -163,7 +163,7 @@ def test_asarray_copy_false(): q = dpctl.SyclQueue() except dpctl.SyclQueueCreationError: pytest.skip("Could not create a queue") - X = dpt.from_numpy(np.random.randn(10, 4), usm_type="device", queue=q) + X = dpt.from_numpy(np.random.randn(10, 4), usm_type="device", sycl_queue=q) Y1 = dpt.asarray(X, copy=False, order="K") assert Y1 is X Y1c = dpt.asarray(X, copy=True, order="K") diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index b5fab57566..7a33e8d87c 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -562,7 +562,7 @@ def test_pyx_capi_check_constants(): def test_tofrom_numpy(shape, dtype, usm_type): q = dpctl.SyclQueue() Xnp = np.zeros(shape, dtype=dtype) - Xusm = dpt.from_numpy(Xnp, usm_type=usm_type, queue=q) + Xusm = dpt.from_numpy(Xnp, usm_type=usm_type, sycl_queue=q) Ynp = np.ones(shape, dtype=dtype) ind = (slice(None, None, None),) * Ynp.ndim Xusm[ind] = Ynp From 755b6b5cd5a35e77674dd6b1c881ef7817f035ce Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 22 Nov 2021 13:35:00 -0600 Subject: [PATCH 023/229] Cleanup/examples (#681) * addressed deprecation warnings in examples/cython/sycl_buffer/use_sycl_buffer.cpp * addressed deprecaton warnings in examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp * Do not use numpy.distutils * Simplify setup by using setuptools alone (no numpy.distutils) * Do not use numpy.distutils, use setuptools instead * Streamlined example using newer syntax --- examples/cython/sycl_buffer/.gitignore | 3 + examples/cython/sycl_buffer/setup.py | 116 +++++++---------- .../cython/sycl_buffer/use_sycl_buffer.cpp | 8 +- .../cython/sycl_direct_linkage/.gitignore | 3 + examples/cython/sycl_direct_linkage/setup.py | 120 ++++++++---------- examples/cython/usm_memory/.gitignore | 3 + examples/cython/usm_memory/setup.py | 115 +++++++---------- .../use_dpctl_syclqueue/pybind11_example.cpp | 2 +- examples/python/usm_memory_allocation.py | 4 +- 9 files changed, 157 insertions(+), 217 deletions(-) create mode 100644 examples/cython/sycl_buffer/.gitignore create mode 100644 examples/cython/sycl_direct_linkage/.gitignore create mode 100644 examples/cython/usm_memory/.gitignore diff --git a/examples/cython/sycl_buffer/.gitignore b/examples/cython/sycl_buffer/.gitignore new file mode 100644 index 0000000000..201efb635e --- /dev/null +++ b/examples/cython/sycl_buffer/.gitignore @@ -0,0 +1,3 @@ +_buffer_example.cpp +*.cpython*.so +*~ diff --git a/examples/cython/sycl_buffer/setup.py b/examples/cython/sycl_buffer/setup.py index 188fdda605..8efce46d7a 100644 --- a/examples/cython/sycl_buffer/setup.py +++ b/examples/cython/sycl_buffer/setup.py @@ -14,74 +14,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="syclbuffer", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on host allocated NumPy array using + SYCL buffers by calling oneMKL functions. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="syclbuffer", + sources=[ + "_buffer_example.pyx", + "use_sycl_buffer.cpp", + ], + include_dirs=[".", np.get_include(), dpctl.get_include()], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="syclbuffer", - sources=[ - join(pdir, "_buffer_example.pyx"), - join(wdir, "use_sycl_buffer.cpp"), - join(wdir, "use_sycl_buffer.h"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.cpp b/examples/cython/sycl_buffer/use_sycl_buffer.cpp index 6abae0e73b..57b28de1fb 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.cpp +++ b/examples/cython/sycl_buffer/use_sycl_buffer.cpp @@ -123,13 +123,13 @@ int c_columnwise_total_no_mkl(DPCTLSyclQueueRef q_ref, sycl::nd_range<2>(global, local), [=](sycl::nd_item<2> it) { size_t i = it.get_global_id(0); size_t j = it.get_global_id(1); - double group_sum = sycl::ONEAPI::reduce( + double group_sum = sycl::reduce_over_group( it.get_group(), (i < n) ? mat_acc[it.get_global_id()] : 0.0, std::plus()); if (it.get_local_id(0) == 0) { - sycl::ONEAPI::atomic_ref< - double, sycl::ONEAPI::memory_order::relaxed, - sycl::ONEAPI::memory_scope::system, + sycl::ext::oneapi::atomic_ref< + double, sycl::ext::oneapi::memory_order::relaxed, + sycl::ext::oneapi::memory_scope::system, sycl::access::address_space::global_space>(ct_acc[j]) += group_sum; } diff --git a/examples/cython/sycl_direct_linkage/.gitignore b/examples/cython/sycl_direct_linkage/.gitignore new file mode 100644 index 0000000000..201efb635e --- /dev/null +++ b/examples/cython/sycl_direct_linkage/.gitignore @@ -0,0 +1,3 @@ +_buffer_example.cpp +*.cpython*.so +*~ diff --git a/examples/cython/sycl_direct_linkage/setup.py b/examples/cython/sycl_direct_linkage/setup.py index fe71d28a0a..31c1e92162 100644 --- a/examples/cython/sycl_direct_linkage/setup.py +++ b/examples/cython/sycl_direct_linkage/setup.py @@ -14,74 +14,54 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="syclbuffer", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on host allocated NumPy array using + SYCL buffers by calling oneMKL functions. + + This extension create SYCL queue in the scope of the function call + incurring large performance overhead. See `sycl_buffer/` example, + where user-constructed `dpctl.SyclQueue` can be given by the user. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="syclbuffer_naive", + sources=[ + "_buffer_example.pyx", + "sycl_function.cpp", + ], + include_dirs=[".", np.get_include(), dpctl.get_include()], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="syclbuffer_naive", - sources=[ - join(pdir, "_buffer_example.pyx"), - join(pdir, "sycl_function.cpp"), - join(pdir, "sycl_function.hpp"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/cython/usm_memory/.gitignore b/examples/cython/usm_memory/.gitignore new file mode 100644 index 0000000000..d423caa640 --- /dev/null +++ b/examples/cython/usm_memory/.gitignore @@ -0,0 +1,3 @@ +blackscholes.cpp +*~ +*.cpython*.so diff --git a/examples/cython/usm_memory/setup.py b/examples/cython/usm_memory/setup.py index a2a507f25b..008370fd62 100644 --- a/examples/cython/usm_memory/setup.py +++ b/examples/cython/usm_memory/setup.py @@ -14,74 +14,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="blackscholes_usm", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on usm allocations. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="blackscholes_usm", + sources=[ + "blackscholes.pyx", + "sycl_blackscholes.cpp", + ], + include_dirs=[".", np.get_include(), dpctl.get_include()], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="blackscholes_usm", - sources=[ - join(pdir, "blackscholes.pyx"), - join(wdir, "sycl_blackscholes.cpp"), - join(wdir, "sycl_blackscholes.hpp"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp b/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp index b90697a4f8..ed22daec2a 100644 --- a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp +++ b/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp @@ -130,7 +130,7 @@ offloaded_array_mod(py::object queue, ->submit([&](sycl::handler &cgh) { sycl::accessor a_acc(a_buf, cgh, sycl::read_only); sycl::accessor r_acc(r_buf, cgh, sycl::write_only, - sycl::noinit); + sycl::no_init); cgh.parallel_for(sycl::range<1>(n), [=](sycl::id<1> idx) { r_acc[idx] = a_acc[idx] % mod; diff --git a/examples/python/usm_memory_allocation.py b/examples/python/usm_memory_allocation.py index 5a54d117d4..caa1d456ac 100644 --- a/examples/python/usm_memory_allocation.py +++ b/examples/python/usm_memory_allocation.py @@ -34,8 +34,8 @@ # allocate using given queue, # i.e. on the device and bound to the context stored in the queue -mdq = dpmem.MemoryUSMDevice(256, queue=mda._queue) +mdq = dpmem.MemoryUSMDevice(256, queue=mda.sycl_queue) # information about device associate with USM buffer print("Allocation performed on device:") -mda._queue.sycl_device.print_device_info() +mda.sycl_queue.print_device_info() From c2abd2c2716fecfdb520b950e82f2f5b4ef5d111 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 11 Nov 2021 08:20:50 -0600 Subject: [PATCH 024/229] Added tensor._dlpack extension, vendored dlpack header Added dlpack's license file in include/dlpack/ Added note in README.md about the location of the origin LICENSE file from which the copy was made Add _dlpack.pyx to flake exception file Implemented to_dlpack_capsule/from_dlpack_capsule functions --- .flake8 | 1 + dpctl/tensor/_dlpack.pxd | 33 ++ dpctl/tensor/_dlpack.pyx | 309 ++++++++++++++++++ dpctl/tensor/include/dlpack/.clang-format | 2 + .../tensor/include/dlpack/LICENSE.third-party | 201 ++++++++++++ dpctl/tensor/include/dlpack/README.md | 7 + dpctl/tensor/include/dlpack/dlpack.h | 213 ++++++++++++ setup.py | 18 + 8 files changed, 784 insertions(+) create mode 100644 dpctl/tensor/_dlpack.pxd create mode 100644 dpctl/tensor/_dlpack.pyx create mode 100644 dpctl/tensor/include/dlpack/.clang-format create mode 100644 dpctl/tensor/include/dlpack/LICENSE.third-party create mode 100644 dpctl/tensor/include/dlpack/README.md create mode 100644 dpctl/tensor/include/dlpack/dlpack.h diff --git a/.flake8 b/.flake8 index 6e9c78b236..c4dc50e407 100644 --- a/.flake8 +++ b/.flake8 @@ -23,6 +23,7 @@ per-file-ignores = dpctl/memory/_memory.pyx: E999, E225, E226, E227 dpctl/program/_program.pyx: E999, E225, E226, E227 dpctl/tensor/_usmarray.pyx: E999, E225, E226, E227 + dpctl/tensor/_dlpack.pyx: E999, E225, E226, E227 dpctl/tensor/numpy_usm_shared.py: F821 dpctl/tests/_cython_api.pyx: E999, E225, E227, E402 dpctl/utils/_compute_follows_data.pyx: E999, E225, E227 diff --git a/dpctl/tensor/_dlpack.pxd b/dpctl/tensor/_dlpack.pxd new file mode 100644 index 0000000000..2a8ff3dda6 --- /dev/null +++ b/dpctl/tensor/_dlpack.pxd @@ -0,0 +1,33 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +from ._usmarray cimport usm_ndarray + + +cdef extern from './include/dlpack/dlpack.h' nogil: + int device_CPU 'kDLCPU' + int device_oneAPI 'kDLOneAPI' + int device_OpenCL 'kDLOpenCL' + + +cpdef object to_dlpack_capsule(usm_ndarray array) except + +cpdef usm_ndarray from_dlpack_capsule(object dltensor) except + + +cpdef from_dlpack(array) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx new file mode 100644 index 0000000000..09887dcb75 --- /dev/null +++ b/dpctl/tensor/_dlpack.pyx @@ -0,0 +1,309 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +cimport cpython +from libc cimport stdlib +from libc.stdint cimport int32_t, int64_t, uint8_t, uint16_t, uint64_t + +cimport dpctl as c_dpctl +cimport dpctl.memory as c_dpmem + +from .._backend cimport ( + DPCTLDevice_Delete, + DPCTLDevice_GetParentDevice, + DPCTLSyclDeviceRef, + DPCTLSyclUSMRef, +) +from ._usmarray cimport usm_ndarray + +import numpy as np + +import dpctl + + +cdef extern from './include/dlpack/dlpack.h' nogil: + cdef int DLPACK_VERSION + + cdef enum DLDeviceType: + kDLCPU + kDLCUDA + kDLCUDAHost + kDLCUDAManaged + kDLROCM + kDLROCMHost + kDLOpenCL + kDLVulkan + kDLMetal + kDLVPI + kDLOneAPI + + ctypedef struct DLDevice: + DLDeviceType device_type + int device_id + + cdef enum DLDataTypeCode: + kDLInt + kDLUInt + kDLFloat + kDLBfloat + kDLComplex + + ctypedef struct DLDataType: + uint8_t code + uint8_t bits + uint16_t lanes + + ctypedef struct DLTensor: + void* data + DLDevice device + int ndim + DLDataType dtype + int64_t* shape + int64_t* strides + uint64_t byte_offset + + ctypedef struct DLManagedTensor: + DLTensor dl_tensor + void* manager_ctx + void (*deleter)(DLManagedTensor*) # noqa: E211 + + +def get_build_dlpack_version(): + return str(DLPACK_VERSION) + + +cdef void pycapsule_deleter(object dlt_capsule): + cdef DLManagedTensor *dlm_tensor = NULL + if cpython.PyCapsule_IsValid(dlt_capsule, 'dltensor'): + dlm_tensor = cpython.PyCapsule_GetPointer( + dlt_capsule, 'dltensor') + dlm_tensor.deleter(dlm_tensor) + elif cpython.PyCapsule_IsValid(dlt_capsule, 'used_dltensor'): + dlm_tensor = cpython.PyCapsule_GetPointer( + dlt_capsule, 'used_dltensor') + dlm_tensor.deleter(dlm_tensor) + + +cdef void managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: + if dlm_tensor is not NULL: + stdlib.free(dlm_tensor.dl_tensor.shape) + cpython.Py_DECREF(dlm_tensor.manager_ctx) + dlm_tensor.manager_ctx = NULL + stdlib.free(dlm_tensor) + + +cdef class DLPackCreationError(Exception): + """ + A DLPackCreateError exception is raised when constructing + DLPack capsule from `usm_ndarray` based on a USM allocation + on a partitioned SYCL device. + """ + pass + + +cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: + """Constructs named Python capsule object referencing + instance of `DLManagerTensor` from `usm_ndarray` instance""" + cdef c_dpctl.SyclQueue ary_sycl_queue + cdef c_dpctl.SyclDevice ary_sycl_device + cdef DPCTLSyclDeviceRef pDRef = NULL + cdef DLManagedTensor *dlm_tensor = NULL + cdef DLTensor* dl_tensor = NULL + cdef int nd = usm_ary.get_ndim() + cdef char* data_ptr = usm_ary.get_data() + cdef Py_ssize_t *shape_ptr = NULL + cdef Py_ssize_t *strides_ptr = NULL + cdef int64_t *shape_strides_ptr = NULL + cdef int i = 0 + cdef int device_id = -1 + + ary_sycl_queue = usm_ary.get_sycl_queue() + ary_sycl_device = ary_sycl_queue.get_sycl_device() + + # check that ary_sycl_device is a non-partitioned device + pDRef = DPCTLDevice_GetParentDevice(ary_sycl_device.get_device_ref()) + if pDRef is not NULL: + DPCTLDevice_Delete(pDRef) + raise DLPackCreationError( + "to_dlpack_capsule: DLPack can only export arrays allocated on " + "non-partitioned SYCL devices." + ) + + dlm_tensor = stdlib.malloc( + sizeof(DLManagedTensor)) + if dlm_tensor is NULL: + raise MemoryError( + "to_dlpack_capsule: Could not allocate memory for DLManagedTensor" + ) + shape_strides_ptr = stdlib.malloc((sizeof(int64_t) * 2) * nd) + if shape_strides_ptr is NULL: + stdlib.free(dlm_tensor) + raise MemoryError( + "to_dlpack_capsule: Could not allocate memory for shape/strides" + ) + shape_ptr = usm_ary.get_shape() + for i in range(nd): + shape_strides_ptr[i] = shape_ptr[i] + strides_ptr = usm_ary.get_strides() + if strides_ptr: + for i in range(nd): + shape_strides_ptr[nd + i] = strides_ptr[i] + + device_id = ary_sycl_device.get_overall_ordinal() + if device_id < 0: + stdlib.free(shape_strides_ptr) + stdlib.free(dlm_tensor) + raise DLPackCreationError( + "to_dlpack_capsule: failed to determine device_id" + ) + + ary_dt = usm_ary.dtype + ary_dtk = ary_dt.kind + + dl_tensor = &dlm_tensor.dl_tensor + dl_tensor.data = data_ptr + dl_tensor.ndim = nd + dl_tensor.byte_offset = 0 + dl_tensor.shape = &shape_strides_ptr[0] + if strides_ptr is NULL: + dl_tensor.strides = NULL + else: + dl_tensor.strides = &shape_strides_ptr[nd] + dl_tensor.device.device_type = kDLOneAPI + dl_tensor.device.device_id = device_id + dl_tensor.dtype.lanes = 1 + dl_tensor.dtype.bits = (ary_dt.itemsize * 8) + if (ary_dtk == "b"): + dl_tensor.dtype.code = kDLUInt + elif (ary_dtk == "u"): + dl_tensor.dtype.code = kDLUInt + elif (ary_dtk == "i"): + dl_tensor.dtype.code = kDLInt + elif (ary_dtk == "f"): + dl_tensor.dtype.code = kDLFloat + elif (ary_dtk == "c"): + dl_tensor.dtype.code = kDLComplex + else: + stdlib.free(shape_strides_ptr) + stdlib.free(dlm_tensor) + raise ValueError("Unrecognized array data type") + + dlm_tensor.manager_ctx = usm_ary + cpython.Py_INCREF(usm_ary) + dlm_tensor.deleter = managed_tensor_deleter + + return cpython.PyCapsule_New(dlm_tensor, 'dltensor', pycapsule_deleter) + + +cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: + """Reconstructs instance of usm_ndarray from named Python + capsule object referencing instance of `DLManagedTensor` without + a copy""" + cdef DLManagedTensor *dlm_tensor = NULL + cdef bytes usm_type + cdef size_t sz = 1 + cdef int i + cdef int element_bytesize = 0 + + if not cpython.PyCapsule_IsValid(py_caps, 'dltensor'): + if cpython.PyCapsule_IsValid(py_caps, 'used_dltensor'): + raise ValueError( + "A DLPack tensor object can not be consumed multiple times" + ) + else: + raise TypeError( + f"A Python 'dltensor' capsule was expected, " + "got {type(dlm_tensor)}" + ) + dlm_tensor = cpython.PyCapsule_GetPointer( + py_caps, "dltensor") + # Verify that we can work with this device + if dlm_tensor.dl_tensor.device.device_type == kDLOneAPI: + q = dpctl.SyclQueue(str(dlm_tensor.dl_tensor.device.device_id)) + usm_type = c_dpmem._Memory.get_pointer_type( + dlm_tensor.dl_tensor.data, + q.sycl_context) + if usm_type == b"unknown": + raise ValueError( + f"Data pointer in DLPack is not bound to default sycl " + "context of device '{device_id}', translated to " + "{q.sycl_device.filter_string}" + ) + if dlm_tensor.dl_tensor.dtype.bits % 8: + raise ValueError( + "Can not import DLPack tensor whose element's " + "bitsize is not a multiple of 8" + ) + if dlm_tensor.dl_tensor.dtype.lanes != 1: + raise ValueError( + "Can not import DLPack tensor with lanes != 1" + ) + for i in range(dlm_tensor.dl_tensor.ndim): + sz = sz * dlm_tensor.dl_tensor.shape[i] + + element_bytesize = (dlm_tensor.dl_tensor.dtype.bits // 8) + sz = sz * element_bytesize + usm_mem = c_dpmem._Memory.create_from_usm_pointer_size_qref( + dlm_tensor.dl_tensor.data, + sz, + (q).get_queue_ref(), + memory_owner=py_caps + ) + py_shape = list() + for i in range(dlm_tensor.dl_tensor.ndim): + py_shape.append(dlm_tensor.dl_tensor.shape[i]) + if (dlm_tensor.dl_tensor.strides is NULL): + py_strides = None + else: + py_strides = list() + for i in range(dlm_tensor.dl_tensor.ndim): + py_strides.append(dlm_tensor.dl_tensor.strides[i]) + if (dlm_tensor.dl_tensor.dtype.code == kDLUInt): + ary_dt = np.dtype("u" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLInt): + ary_dt = np.dtype("i" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLFloat): + ary_dt = np.dtype("f" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLComplex): + ary_dt = np.dtype("c" + str(element_bytesize)) + else: + raise ValueError( + "Can not import DLPack tensor with type code {}.".format( + dlm_tensor.dl_tensor.dtype.code + ) + ) + res_ary = usm_ndarray( + py_shape, + dtype=ary_dt, + buffer=usm_mem, + strides=py_strides + ) + cpython.PyCapsule_SetName(py_caps, 'used_dltensor') + return res_ary + else: + raise ValueError( + "The DLPack tensor resides on unsupported device." + ) + + +cpdef from_dlpack(array): + """Constructs `usm_ndarray` from a Python object that implements + `__dlpack__` protocol.""" + pass diff --git a/dpctl/tensor/include/dlpack/.clang-format b/dpctl/tensor/include/dlpack/.clang-format new file mode 100644 index 0000000000..9d159247d5 --- /dev/null +++ b/dpctl/tensor/include/dlpack/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: false diff --git a/dpctl/tensor/include/dlpack/LICENSE.third-party b/dpctl/tensor/include/dlpack/LICENSE.third-party new file mode 100644 index 0000000000..20a9c8a7b4 --- /dev/null +++ b/dpctl/tensor/include/dlpack/LICENSE.third-party @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dpctl/tensor/include/dlpack/README.md b/dpctl/tensor/include/dlpack/README.md new file mode 100644 index 0000000000..5d9bf51177 --- /dev/null +++ b/dpctl/tensor/include/dlpack/README.md @@ -0,0 +1,7 @@ +# DLPack header + +The header `dlpack.h` downloaded from `https://github.com/dmlc/dlpack.git` remote at commit [`98861a50e5`](https://github.com/dmlc/dlpack/commit/98861a50e5ade5a6b2df388b12d67b418e3baebe). + +The file can also be viewed using github web interface at https://github.com/dmlc/dlpack/blob/98861a50e5ade5a6b2df388b12d67b418e3baebe/include/dlpack/dlpack.h + +License file was retrived from https://github.com/dmlc/dlpack/blob/main/LICENSE diff --git a/dpctl/tensor/include/dlpack/dlpack.h b/dpctl/tensor/include/dlpack/dlpack.h new file mode 100644 index 0000000000..afbac0573a --- /dev/null +++ b/dpctl/tensor/include/dlpack/dlpack.h @@ -0,0 +1,213 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file dlpack.h + * \brief The common header of DLPack. + */ +#ifndef DLPACK_DLPACK_H_ +#define DLPACK_DLPACK_H_ + +#ifdef __cplusplus +#define DLPACK_EXTERN_C extern "C" +#else +#define DLPACK_EXTERN_C +#endif + +/*! \brief The current version of dlpack */ +#define DLPACK_VERSION 60 + +/*! \brief DLPACK_DLL prefix for windows */ +#ifdef _WIN32 +#ifdef DLPACK_EXPORTS +#define DLPACK_DLL __declspec(dllexport) +#else +#define DLPACK_DLL __declspec(dllimport) +#endif +#else +#define DLPACK_DLL +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +/*! + * \brief The device type in DLDevice. + */ +typedef enum { + /*! \brief CPU device */ + kDLCPU = 1, + /*! \brief CUDA GPU device */ + kDLCUDA = 2, + /*! + * \brief Pinned CUDA CPU memory by cudaMallocHost + */ + kDLCUDAHost = 3, + /*! \brief OpenCL devices. */ + kDLOpenCL = 4, + /*! \brief Vulkan buffer for next generation graphics. */ + kDLVulkan = 7, + /*! \brief Metal for Apple GPU. */ + kDLMetal = 8, + /*! \brief Verilog simulator buffer */ + kDLVPI = 9, + /*! \brief ROCm GPUs for AMD GPUs */ + kDLROCM = 10, + /*! + * \brief Pinned ROCm CPU memory allocated by hipMallocHost + */ + kDLROCMHost = 11, + /*! + * \brief Reserved extension device type, + * used for quickly test extension device + * The semantics can differ depending on the implementation. + */ + kDLExtDev = 12, + /*! + * \brief CUDA managed/unified memory allocated by cudaMallocManaged + */ + kDLCUDAManaged = 13, + /*! + * \brief Unified shared memory allocated on a oneAPI non-partititioned + * device. Call to oneAPI runtime is required to determine the device + * type, the USM allocation type and the sycl context it is bound to. + * + */ + kDLOneAPI = 14, +} DLDeviceType; + +/*! + * \brief A Device for Tensor and operator. + */ +typedef struct { + /*! \brief The device type used in the device. */ + DLDeviceType device_type; + /*! + * \brief The device index. + * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0. + */ + int device_id; +} DLDevice; + +/*! + * \brief The type code options DLDataType. + */ +typedef enum { + /*! \brief signed integer */ + kDLInt = 0U, + /*! \brief unsigned integer */ + kDLUInt = 1U, + /*! \brief IEEE floating point */ + kDLFloat = 2U, + /*! + * \brief Opaque handle type, reserved for testing purposes. + * Frameworks need to agree on the handle data type for the exchange to be well-defined. + */ + kDLOpaqueHandle = 3U, + /*! \brief bfloat16 */ + kDLBfloat = 4U, + /*! + * \brief complex number + * (C/C++/Python layout: compact struct per complex number) + */ + kDLComplex = 5U, +} DLDataTypeCode; + +/*! + * \brief The data type the tensor can hold. + * + * Examples + * - float: type_code = 2, bits = 32, lanes=1 + * - float4(vectorized 4 float): type_code = 2, bits = 32, lanes=4 + * - int8: type_code = 0, bits = 8, lanes=1 + * - std::complex: type_code = 5, bits = 64, lanes = 1 + */ +typedef struct { + /*! + * \brief Type code of base types. + * We keep it uint8_t instead of DLDataTypeCode for minimal memory + * footprint, but the value should be one of DLDataTypeCode enum values. + * */ + uint8_t code; + /*! + * \brief Number of bits, common choices are 8, 16, 32. + */ + uint8_t bits; + /*! \brief Number of lanes in the type, used for vector types. */ + uint16_t lanes; +} DLDataType; + +/*! + * \brief Plain C Tensor object, does not manage memory. + */ +typedef struct { + /*! + * \brief The data pointer points to the allocated data. This will be CUDA + * device pointer or cl_mem handle in OpenCL. It may be opaque on some device + * types. This pointer is always aligned to 256 bytes as in CUDA. The + * `byte_offset` field should be used to point to the beginning of the data. + * + * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow, + * TVM, perhaps others) do not adhere to this 256 byte aligment requirement + * on CPU/CUDA/ROCm, and always use `byte_offset=0`. This must be fixed + * (after which this note will be updated); at the moment it is recommended + * to not rely on the data pointer being correctly aligned. + * + * For given DLTensor, the size of memory required to store the contents of + * data is calculated as follows: + * + * \code{.c} + * static inline size_t GetDataSize(const DLTensor* t) { + * size_t size = 1; + * for (tvm_index_t i = 0; i < t->ndim; ++i) { + * size *= t->shape[i]; + * } + * size *= (t->dtype.bits * t->dtype.lanes + 7) / 8; + * return size; + * } + * \endcode + */ + void* data; + /*! \brief The device of the tensor */ + DLDevice device; + /*! \brief Number of dimensions */ + int ndim; + /*! \brief The data type of the pointer*/ + DLDataType dtype; + /*! \brief The shape of the tensor */ + int64_t* shape; + /*! + * \brief strides of the tensor (in number of elements, not bytes) + * can be NULL, indicating tensor is compact and row-majored. + */ + int64_t* strides; + /*! \brief The offset in bytes to the beginning pointer to data */ + uint64_t byte_offset; +} DLTensor; + +/*! + * \brief C Tensor object, manage memory of DLTensor. This data structure is + * intended to facilitate the borrowing of DLTensor by another framework. It is + * not meant to transfer the tensor. When the borrowing framework doesn't need + * the tensor, it should call the deleter to notify the host that the resource + * is no longer needed. + */ +typedef struct DLManagedTensor { + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; + /*! \brief the context of the original host framework of DLManagedTensor in + * which DLManagedTensor is used in the framework. It can also be NULL. + */ + void * manager_ctx; + /*! \brief Destructor signature void (*)(void*) - this should be called + * to destruct manager_ctx which holds the DLManagedTensor. It can be NULL + * if there is no way for the caller to provide a reasonable destructor. + * The destructors deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensor * self); +} DLManagedTensor; +#ifdef __cplusplus +} // DLPACK_EXTERN_C +#endif +#endif // DLPACK_DLPACK_H_ diff --git a/setup.py b/setup.py index 928a8e0010..5435f3d489 100644 --- a/setup.py +++ b/setup.py @@ -240,6 +240,24 @@ def extensions(): runtime_library_dirs=extension_args["runtime_library_dirs"], define_macros=extension_args["define_macros"], ), + Extension( + "dpctl.tensor._dlpack", + [ + os.path.join("dpctl", "tensor", "_dlpack.pyx"), + ], + depends=extension_args["depends"], + language="c++", + include_dirs=extension_args["include_dirs"] + + [ + os.path.join("dpctl", "tensor"), + ], + extra_compile_args=extension_args["extra_compile_args"], + extra_link_args=extension_args["extra_link_args"], + libraries=extension_args["libraries"], + library_dirs=extension_args["library_dirs"], + runtime_library_dirs=extension_args["runtime_library_dirs"], + define_macros=extension_args["define_macros"], + ), ] return extensions From fff470dbf64aff05141bdd44e92dfb7fc99a1836 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 15 Nov 2021 15:20:48 -0600 Subject: [PATCH 025/229] Implemented usm_ndarray.__dlpack__, usm_ndarray.__dlpack_device__ --- dpctl/tensor/_usmarray.pyx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 57c5225a3e..4b4d41dd87 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -32,6 +32,7 @@ from cpython.tuple cimport PyTuple_New, PyTuple_SetItem cimport dpctl as c_dpctl cimport dpctl.memory as c_dpmem +cimport dpctl.tensor._dlpack as c_dlpack include "_stride_utils.pxi" include "_types.pxi" @@ -738,10 +739,20 @@ cdef class usm_ndarray: return NotImplemented def __dlpack__(self, stream=None): - return NotImplemented + """Produce DLPack capsule""" + if stream is None: + return c_dlpack.to_dlpack_capsule(self) + else: + raise NotImplementedError( + "Only stream=None is supported. " + "Use `dpctl.SyclQueue.submit_barrier` to synchronize queues." + ) def __dlpack_device__(self): - return NotImplemented + return ( + c_dlpack.device_oneAPI, + (self.sycl_device).get_overall_ordinal(), + ) def __eq__(self, other): return _dispatch_binary_elementwise(self, "equal", other) From 3a96289d8a16065ed5d87f3f01ff31f242fadb3e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 15 Nov 2021 15:46:23 -0600 Subject: [PATCH 026/229] Implemented from_dlpack(array) --- dpctl/tensor/_dlpack.pyx | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 09887dcb75..35cb8140fa 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -305,5 +305,18 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: cpdef from_dlpack(array): """Constructs `usm_ndarray` from a Python object that implements - `__dlpack__` protocol.""" - pass + `__dlpack__` protocol. + """ + if not hasattr(array, "__dlpack__"): + raise TypeError( + "The argument of type {type(array)} does not implement " + "`__dlpack__` method." + ) + dlpack_attr = getattr(array, "__dlpack__") + if not callable(dlpack_attr): + raise TypeError( + "The argument of type {type(array)} does not implement " + "`__dlpack__` method." + ) + dlpack_capsule = dlpack_attr() + return from_dlpack_capsule(dlpack_capsule) From 96511159128bca68c417205251443130abda8249 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 15 Nov 2021 15:50:43 -0600 Subject: [PATCH 027/229] Exported dpctl.tensor.from_dlpack --- dpctl/tensor/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dpctl/tensor/__init__.py b/dpctl/tensor/__init__.py index 6036099c62..ab2bf72ebb 100644 --- a/dpctl/tensor/__init__.py +++ b/dpctl/tensor/__init__.py @@ -34,6 +34,7 @@ from dpctl.tensor._copy_utils import copy_to_numpy as asnumpy from dpctl.tensor._copy_utils import copy_to_numpy as to_numpy from dpctl.tensor._ctors import asarray, empty +from dpctl.tensor._dlpack import from_dlpack from dpctl.tensor._reshape import reshape from dpctl.tensor._usmarray import usm_ndarray @@ -47,4 +48,5 @@ "from_numpy", "to_numpy", "asnumpy", + "from_dlpack", ] From a2bf38f787b85e4e1bc82ec6408a0e70181e98d0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 16 Nov 2021 09:48:36 -0600 Subject: [PATCH 028/229] DLManagedTensor lifetime management implemented per array-API specs 1. The pycapsule destructor only calls DLManagedTensor.deleter is the name is "dltensor" 2. Code consuming the DLPack capsule renamed the capsule (to avoid destructor calling the deleter) and instead creates an internal object to do that and uses that internal object as the base of _Memory object `from_dlpack_capsule` function should handle NULL data field For zero-elements arrays in DLPack, allocate 1 element Proper support for strides added. Expanded docstring of `dpctl.tensor.from_dlpack` --- dpctl/tensor/_dlpack.pyx | 107 +++++++++++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 21 deletions(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 35cb8140fa..86daf8b72b 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -36,6 +36,7 @@ from ._usmarray cimport usm_ndarray import numpy as np import dpctl +import dpctl.memory as dpmem cdef extern from './include/dlpack/dlpack.h' nogil: @@ -95,10 +96,6 @@ cdef void pycapsule_deleter(object dlt_capsule): dlm_tensor = cpython.PyCapsule_GetPointer( dlt_capsule, 'dltensor') dlm_tensor.deleter(dlm_tensor) - elif cpython.PyCapsule_IsValid(dlt_capsule, 'used_dltensor'): - dlm_tensor = cpython.PyCapsule_GetPointer( - dlt_capsule, 'used_dltensor') - dlm_tensor.deleter(dlm_tensor) cdef void managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: @@ -133,7 +130,11 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: cdef int64_t *shape_strides_ptr = NULL cdef int i = 0 cdef int device_id = -1 + cdef char* base_ptr = NULL + cdef Py_ssize_t element_offset = 0 + cdef Py_ssize_t byte_offset = 0 + ary_base = usm_ary.get_base() ary_sycl_queue = usm_ary.get_sycl_queue() ary_sycl_device = ary_sycl_queue.get_sycl_device() @@ -176,11 +177,13 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: ary_dt = usm_ary.dtype ary_dtk = ary_dt.kind + element_offset = usm_ary.get_offset() + byte_offset = element_offset * (ary_dt.itemsize) dl_tensor = &dlm_tensor.dl_tensor - dl_tensor.data = data_ptr + dl_tensor.data = (data_ptr - byte_offset) dl_tensor.ndim = nd - dl_tensor.byte_offset = 0 + dl_tensor.byte_offset = byte_offset dl_tensor.shape = &shape_strides_ptr[0] if strides_ptr is NULL: dl_tensor.strides = NULL @@ -212,6 +215,24 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: return cpython.PyCapsule_New(dlm_tensor, 'dltensor', pycapsule_deleter) +cdef class _DLManagedTensorOwner: + """Helper class managing lifetimes of the DLManagedTensor struct""" + cdef DLManagedTensor *dlm_tensor + + def __cinit__(self): + self.dlm_tensor = NULL + + def __dealloc__(self): + if self.dlm_tensor: + self.dlm_tensor.deleter(self.dlm_tensor) + + @staticmethod + cdef _DLManagedTensorOwner _create(DLManagedTensor *dlm_tensor_src): + cdef _DLManagedTensorOwner res = _DLManagedTensorOwner.__new__(_DLManagedTensorOwner) + res.dlm_tensor = dlm_tensor_src + return res + + cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: """Reconstructs instance of usm_ndarray from named Python capsule object referencing instance of `DLManagedTensor` without @@ -221,6 +242,11 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: cdef size_t sz = 1 cdef int i cdef int element_bytesize = 0 + cdef Py_ssize_t offset_min = 0 + cdef Py_ssize_t offset_max = 0 + cdef int64_t stride_i + cdef char* mem_ptr = NULL + cdef Py_ssize_t element_offset = 0 if not cpython.PyCapsule_IsValid(py_caps, 'dltensor'): if cpython.PyCapsule_IsValid(py_caps, 'used_dltensor'): @@ -237,9 +263,12 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: # Verify that we can work with this device if dlm_tensor.dl_tensor.device.device_type == kDLOneAPI: q = dpctl.SyclQueue(str(dlm_tensor.dl_tensor.device.device_id)) - usm_type = c_dpmem._Memory.get_pointer_type( - dlm_tensor.dl_tensor.data, - q.sycl_context) + if dlm_tensor.dl_tensor.data is NULL: + usm_type = b"device" + else: + usm_type = c_dpmem._Memory.get_pointer_type( + dlm_tensor.dl_tensor.data, + q.sycl_context) if usm_type == b"unknown": raise ValueError( f"Data pointer in DLPack is not bound to default sycl " @@ -255,17 +284,45 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: raise ValueError( "Can not import DLPack tensor with lanes != 1" ) - for i in range(dlm_tensor.dl_tensor.ndim): - sz = sz * dlm_tensor.dl_tensor.shape[i] + if dlm_tensor.dl_tensor.strides is NULL: + for i in range(dlm_tensor.dl_tensor.ndim): + sz = sz * dlm_tensor.dl_tensor.shape[i] + else: + offset_min = 0 + offset_max = 0 + for i in range(dlm_tensor.dl_tensor.ndim): + stride_i = dlm_tensor.dl_tensor.strides[i] + if stride_i > 0: + offset_max = offset_max + stride_i * ( + dlm_tensor.dl_tensor.shape[i] - 1 + ) + else: + offset_min = offset_min + stride_i * ( + dlm_tensor.dl_tensor.shape[i] - 1 + ) + sz = offset_max - offset_min + 1 + if sz == 0: + sz = 1 element_bytesize = (dlm_tensor.dl_tensor.dtype.bits // 8) sz = sz * element_bytesize - usm_mem = c_dpmem._Memory.create_from_usm_pointer_size_qref( - dlm_tensor.dl_tensor.data, - sz, - (q).get_queue_ref(), - memory_owner=py_caps - ) + element_offset = dlm_tensor.dl_tensor.byte_offset // element_bytesize + + # transfer dlm_tensor ownership + dlm_holder = _DLManagedTensorOwner._create(dlm_tensor) + cpython.PyCapsule_SetName(py_caps, 'used_dltensor') + + if dlm_tensor.dl_tensor.data is NULL: + usm_mem = dpmem.MemoryUSMDevice(sz, q) + else: + mem_ptr = dlm_tensor.dl_tensor.data + dlm_tensor.dl_tensor.byte_offset + mem_ptr = mem_ptr - (element_offset * element_bytesize) + usm_mem = c_dpmem._Memory.create_from_usm_pointer_size_qref( + mem_ptr, + sz, + (q).get_queue_ref(), + memory_owner=dlm_holder + ) py_shape = list() for i in range(dlm_tensor.dl_tensor.ndim): py_shape.append(dlm_tensor.dl_tensor.shape[i]) @@ -293,9 +350,9 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: py_shape, dtype=ary_dt, buffer=usm_mem, - strides=py_strides + strides=py_strides, + offset=element_offset ) - cpython.PyCapsule_SetName(py_caps, 'used_dltensor') return res_ary else: raise ValueError( @@ -304,8 +361,16 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: cpdef from_dlpack(array): - """Constructs `usm_ndarray` from a Python object that implements - `__dlpack__` protocol. + """dpctl.tensor.from_dlpack(obj) + + Constructs :class:`dpctl.tensor.usm_ndarray` instance from a Python + object `obj` that implements `__dlpack__` protocol. The output + array is always a zero-copy view of the input. + + Raises: + TypeError: if `obj` does not implement `__dlpack__` method. + ValueError: if zero copy view can not be constructed because + the input array resides on an unsupported device. """ if not hasattr(array, "__dlpack__"): raise TypeError( From ee4dcc6567cbc896f8df80be2487ec8a213e4aa4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 16 Nov 2021 09:50:15 -0600 Subject: [PATCH 029/229] Adding test_usm_ndarray_dlpack Test should anticipate that dlpack roundtripping changes bool dtype to uint8 Adding test for `from_dlpack` input validation --- dpctl/tests/test_usm_ndarray_dlpack.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 dpctl/tests/test_usm_ndarray_dlpack.py diff --git a/dpctl/tests/test_usm_ndarray_dlpack.py b/dpctl/tests/test_usm_ndarray_dlpack.py new file mode 100644 index 0000000000..a40688965b --- /dev/null +++ b/dpctl/tests/test_usm_ndarray_dlpack.py @@ -0,0 +1,120 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ctypes + +import pytest + +import dpctl +import dpctl.tensor as dpt + +device_oneAPI = 14 # DLDeviceType.kDLOneAPI + +_usm_types_list = ["shared", "device", "host"] + + +@pytest.fixture(params=_usm_types_list) +def usm_type(request): + return request.param + + +_typestrs_list = [ + "b1", + "u1", + "i1", + "u2", + "i2", + "u4", + "i4", + "u8", + "i8", + "f2", + "f4", + "f8", + "c8", + "c16", +] + + +@pytest.fixture(params=_typestrs_list) +def typestr(request): + return request.param + + +def test_dlpack_device(usm_type): + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty((64,), dtype="u1", usm_type=usm_type, device=sycl_dev) + dev = X.__dlpack_device__() + assert type(dev) is tuple + assert len(dev) == 2 + assert dev[0] == device_oneAPI + assert sycl_dev == all_root_devices[dev[1]] + + +def test_dlpack_exporter(typestr, usm_type): + caps_fn = ctypes.pythonapi.PyCapsule_IsValid + caps_fn.restype = bool + caps_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty((64,), dtype=typestr, usm_type=usm_type, device=sycl_dev) + caps = X.__dlpack__() + assert caps_fn(caps, b"dltensor") + Y = X[::2] + caps2 = Y.__dlpack__() + assert caps_fn(caps2, b"dltensor") + + +@pytest.mark.parametrize("shape", [tuple(), (2,), (3, 0, 1), (2, 2, 2)]) +def test_from_dlpack(shape, typestr, usm_type): + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty(shape, dtype=typestr, usm_type=usm_type, device=sycl_dev) + Y = dpt.from_dlpack(X) + assert X.shape == Y.shape + assert X.dtype == Y.dtype or ( + str(X.dtype) == "bool" and str(Y.dtype) == "uint8" + ) + assert X.sycl_device == Y.sycl_device + assert X.usm_type == Y.usm_type + assert X._pointer == Y._pointer + if Y.ndim: + V = Y[::-1] + W = dpt.from_dlpack(V) + assert V.strides == W.strides + + +def test_from_dlpack_input_validation(): + vstr = dpt._dlpack.get_build_dlpack_version() + assert type(vstr) is str + with pytest.raises(TypeError): + dpt.from_dlpack(None) + + class DummyWithProperty: + @property + def __dlpack__(self): + return None + + with pytest.raises(TypeError): + dpt.from_dlpack(DummyWithProperty()) + + class DummyWithMethod: + def __dlpack__(self): + return None + + with pytest.raises(TypeError): + dpt.from_dlpack(DummyWithMethod()) From ea71e66e0f62caebae954373e96059f126007d48 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sat, 20 Nov 2021 08:15:35 -0600 Subject: [PATCH 030/229] dlpack support noted in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c09fea7af3..371ff0a0f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `dpctl.tensor.asarray`, `dpctl.tensor.empty` implemented (#646). +- `dpctl.tensor.usm_ndarray` adds support for DLPack protocol. `dpctl.tensor.from_dlpack` implemented (#682). ### Changed - dpctl-capi is now renamed to `libsyclinterface` (#666). From 29c2cbc34a82f7007f8e170d9b2548ab3e2b48d4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 21:04:52 -0600 Subject: [PATCH 031/229] Use oclclpuexp and fpgaemu published with 2021-09 release of Intel llvm/sycl (#686) --- .github/workflows/os-llvm-sycl-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index b2101c24bb..92b85e1318 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-20.04 env: - OCLCPUEXP_FN: oclcpuexp-2021.12.6.0.19_rel.tar.gz - FPGAEMU_FN: fpgaemu-2021.12.6.0.19_rel.tar.gz + OCLCPUEXP_FN: oclcpuexp-2021.12.9.0.24_rel.tar.gz + FPGAEMU_FN: fpgaemu-2021.12.9.0.24_rel.tar.gz TBB_FN: oneapi-tbb-2021.4.0-lin.tgz steps: @@ -48,8 +48,8 @@ jobs: export DOWNLOAD_URL_PREFIX=https://github.com/intel/llvm/releases/download rm -rf dpcpp-compiler.tar.gz wget ${DOWNLOAD_URL_PREFIX}/${NIGHTLY_TAG}/dpcpp-compiler.tar.gz && echo ${LATEST_LLVM_TAG_SHA} > bundle_id.txt || rm -rf bundle_id.txt - [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-07/${OCLCPUEXP_FN} || rm -rf bundle_id.txt - [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-07/${FPGAEMU_FN} || rm -rf bundle_id.txt + [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-09/${OCLCPUEXP_FN} || rm -rf bundle_id.txt + [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-09/${FPGAEMU_FN} || rm -rf bundle_id.txt [ -f ${TBB_FN} ] || wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.4.0/${TBB_FN} || rm -rf bundle_id.txt rm -rf dpcpp_compiler tar xf dpcpp-compiler.tar.gz From b6b6b5555263b9f8d0fc80cc9363317ec116e23b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 24 Nov 2021 10:33:33 -0600 Subject: [PATCH 032/229] Addressed PR feedback 1. Added docstrings 2. Exported DLPackCreationError in `_dlpack.pxd` 3. Added validation in `__dlpack_device__` to raise an error if device_id came back -1 (not-found) 4. `to_dlpack_capsule(usm_ary)` raises DLPackCreationError if the array context is not the default context (the one created in dpctl.SyclQueue(dev) call) --- dpctl/tensor/_dlpack.pxd | 8 ++++++ dpctl/tensor/_dlpack.pyx | 50 +++++++++++++++++++++++--------------- dpctl/tensor/_usmarray.pyx | 36 +++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/dpctl/tensor/_dlpack.pxd b/dpctl/tensor/_dlpack.pxd index 2a8ff3dda6..439880f7d2 100644 --- a/dpctl/tensor/_dlpack.pxd +++ b/dpctl/tensor/_dlpack.pxd @@ -31,3 +31,11 @@ cpdef object to_dlpack_capsule(usm_ndarray array) except + cpdef usm_ndarray from_dlpack_capsule(object dltensor) except + cpdef from_dlpack(array) + +cdef class DLPackCreationError(Exception): + """ + A DLPackCreateError exception is raised when constructing + DLPack capsule from `usm_ndarray` based on a USM allocation + on a partitioned SYCL device. + """ + pass diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 86daf8b72b..5cf799521d 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -87,10 +87,14 @@ cdef extern from './include/dlpack/dlpack.h' nogil: def get_build_dlpack_version(): + """ + Returns the string value of DLPACK_VERSION from dlpack.h + `dpcl.tensor` was built with. + """ return str(DLPACK_VERSION) -cdef void pycapsule_deleter(object dlt_capsule): +cdef void _pycapsule_deleter(object dlt_capsule): cdef DLManagedTensor *dlm_tensor = NULL if cpython.PyCapsule_IsValid(dlt_capsule, 'dltensor'): dlm_tensor = cpython.PyCapsule_GetPointer( @@ -98,7 +102,7 @@ cdef void pycapsule_deleter(object dlt_capsule): dlm_tensor.deleter(dlm_tensor) -cdef void managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: +cdef void _managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: if dlm_tensor is not NULL: stdlib.free(dlm_tensor.dl_tensor.shape) cpython.Py_DECREF(dlm_tensor.manager_ctx) @@ -106,18 +110,12 @@ cdef void managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: stdlib.free(dlm_tensor) -cdef class DLPackCreationError(Exception): +cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: """ - A DLPackCreateError exception is raised when constructing - DLPack capsule from `usm_ndarray` based on a USM allocation - on a partitioned SYCL device. + Constructs named Python capsule object referencing + instance of `DLManagerTensor` from + :class:`dpctl.tensor.usm_ndarray` instance. """ - pass - - -cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: - """Constructs named Python capsule object referencing - instance of `DLManagerTensor` from `usm_ndarray` instance""" cdef c_dpctl.SyclQueue ary_sycl_queue cdef c_dpctl.SyclDevice ary_sycl_device cdef DPCTLSyclDeviceRef pDRef = NULL @@ -146,6 +144,14 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: "to_dlpack_capsule: DLPack can only export arrays allocated on " "non-partitioned SYCL devices." ) + # TODO: check that ary_sycl_context is the default context + default_context = dpctl.SyclQueue(ary_sycl_device).sycl_context + if not usm_ary.sycl_context == default_context: + raise DLPackCreationError( + "to_dlpack_capsule: DLPack can only export arrays based on USM " + "allocations bound to a default platform SYCL context" + ) + dlm_tensor = stdlib.malloc( sizeof(DLManagedTensor)) @@ -210,13 +216,16 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: dlm_tensor.manager_ctx = usm_ary cpython.Py_INCREF(usm_ary) - dlm_tensor.deleter = managed_tensor_deleter + dlm_tensor.deleter = _managed_tensor_deleter - return cpython.PyCapsule_New(dlm_tensor, 'dltensor', pycapsule_deleter) + return cpython.PyCapsule_New(dlm_tensor, 'dltensor', _pycapsule_deleter) cdef class _DLManagedTensorOwner: - """Helper class managing lifetimes of the DLManagedTensor struct""" + """ + Helper class managing the lifetime of the DLManagedTensor struct + transferred from a 'dlpack' capsule. + """ cdef DLManagedTensor *dlm_tensor def __cinit__(self): @@ -234,9 +243,11 @@ cdef class _DLManagedTensorOwner: cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: - """Reconstructs instance of usm_ndarray from named Python - capsule object referencing instance of `DLManagedTensor` without - a copy""" + """ + Reconstructs instance of :class:`dpctl.tensor.usm_ndarray` from + named Python capsule object referencing instance of `DLManagedTensor` + without copy. The instance forms a view in the memory of the tensor. + """ cdef DLManagedTensor *dlm_tensor = NULL cdef bytes usm_type cdef size_t sz = 1 @@ -361,7 +372,8 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: cpdef from_dlpack(array): - """dpctl.tensor.from_dlpack(obj) + """ + dpctl.tensor.from_dlpack(obj) Constructs :class:`dpctl.tensor.usm_ndarray` instance from a Python object `obj` that implements `__dlpack__` protocol. The output diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 4b4d41dd87..75131fd916 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -739,7 +739,16 @@ cdef class usm_ndarray: return NotImplemented def __dlpack__(self, stream=None): - """Produce DLPack capsule""" + """ + Produces DLPack capsule. + + Raises: + MemoryError: when host memory can not be allocated. + DLPackCreationError: when array is allocated on a partitioned + SYCL device, or with a non-default context. + NotImplementedError: when non-default value of `stream` keyword + is used. + """ if stream is None: return c_dlpack.to_dlpack_capsule(self) else: @@ -749,10 +758,27 @@ cdef class usm_ndarray: ) def __dlpack_device__(self): - return ( - c_dlpack.device_oneAPI, - (self.sycl_device).get_overall_ordinal(), - ) + """ + Gives a tuple (`device_type`, `device_id`) corresponding to `DLDevice` + entry in `DLTensor` in DLPack protocol. + + The tuple describes the non-partitioned device where the array + has been allocated. + + Raises: + DLPackCreationError: when array is allocation on a partitioned + SYCL device + """ + cdef int dev_id = (self.sycl_device).get_overall_ordinal() + if dev_id < 0: + raise c_dlpack.DLPackCreationError( + "DLPack protocol is only supported for non-partitioned devices" + ) + else: + return ( + c_dlpack.device_oneAPI, + dev_id, + ) def __eq__(self, other): return _dispatch_binary_elementwise(self, "equal", other) From 4c23947a85a3ce897e303dc44c98928cfced9d3f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 25 Nov 2021 11:35:03 -0600 Subject: [PATCH 033/229] Make sure that dpctl/tensor/include/dlpack is included in the layout --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 19f37e5a30..5ce66287a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include versioneer.py recursive-include dpctl/include *.h +recursive-include dpctl/tensor/include * recursive-include dpctl *.pxd include dpctl/_sycl_context.h include dpctl/_sycl_context_api.h From 9163f168251937591b921a13c7337a07dd4c0773 Mon Sep 17 00:00:00 2001 From: "elena.totmenina" Date: Fri, 26 Nov 2021 18:32:36 +0300 Subject: [PATCH 034/229] Fix upload condition --- .github/workflows/conda-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 4e72d54517..86d9a025ed 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -215,7 +215,7 @@ jobs: upload_linux: needs: test_linux - if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true)}} + if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')}} runs-on: ubuntu-latest strategy: matrix: @@ -240,7 +240,7 @@ jobs: upload_windows: needs: test_windows - if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true)}} + if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')}} runs-on: windows-latest strategy: matrix: From e76953e9e2bbfd12c7e468163de7d43e5e1d97e1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 29 Nov 2021 10:24:53 -0600 Subject: [PATCH 035/229] array_info_sequence should handle empty sequences as host data Added a test for this input --- dpctl/tensor/_ctors.py | 3 +++ dpctl/tests/test_tensor_asarray.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index d0f459d1c5..72b28f1238 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -63,6 +63,9 @@ def _array_info_sequence(li): raise ValueError( "Inconsistent dimensions, {} and {}".format(dim, el_dim) ) + if dim is None: + dim = tuple() + device = _host_set return (n,) + dim, dt, device diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py index c7734b8194..c39471497b 100644 --- a/dpctl/tests/test_tensor_asarray.py +++ b/dpctl/tests/test_tensor_asarray.py @@ -80,6 +80,12 @@ def test_asarray_from_sequence(): Y = dpt.asarray(X, usm_type="device") assert type(Y) is dpt.usm_ndarray assert Y.ndim == 2 + assert Y.shape == (len(X), 2) + + X = [] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == (0,) def test_asarray_from_object_with_suai(): From 5f279d6142265f992f22d5bc65c17e72137f42e8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 29 Nov 2021 10:50:35 -0600 Subject: [PATCH 036/229] Fixes #690 Since bool is a subclass of int, the lack of missing test for instance of bool was being handled by test for instance of int. Fixed implementation, added a test. --- dpctl/tensor/_ctors.py | 2 ++ dpctl/tests/test_tensor_asarray.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index 72b28f1238..5126f40837 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -32,6 +32,8 @@ def _array_info_dispatch(obj): return obj.shape, obj.dtype, _host_set elif isinstance(obj, range): return (len(obj),), int, _host_set + elif isinstance(obj, bool): + return _empty_tuple, bool, _host_set elif isinstance(obj, float): return _empty_tuple, float, _host_set elif isinstance(obj, int): diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py index c39471497b..8c61fc9f13 100644 --- a/dpctl/tests/test_tensor_asarray.py +++ b/dpctl/tests/test_tensor_asarray.py @@ -87,6 +87,11 @@ def test_asarray_from_sequence(): assert type(Y) is dpt.usm_ndarray assert Y.shape == (0,) + X = [True, False] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.dtype.kind == "b" + def test_asarray_from_object_with_suai(): """Test that asarray can deal with opaque objects implementing SUAI""" From cd8536971a532ae6665603bc4730a4b4e5309213 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 06:51:01 -0600 Subject: [PATCH 037/229] Merge pull request #695 from IntelPython/gh693-reshape-clears-contiguity-flag Fixes #693 : Reshape clears contiguity flags --- dpctl/tensor/_stride_utils.pxi | 54 ++++++++++++++++++++-------- dpctl/tests/test_usm_ndarray_ctor.py | 17 +++++++-- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/dpctl/tensor/_stride_utils.pxi b/dpctl/tensor/_stride_utils.pxi index 37d5a366b7..c019cd8ddd 100644 --- a/dpctl/tensor/_stride_utils.pxi +++ b/dpctl/tensor/_stride_utils.pxi @@ -61,6 +61,7 @@ cdef int _from_input_shape_strides( Otherwise they are set to NULL """ cdef int i + cdef int j cdef int all_incr = 1 cdef int all_decr = 1 cdef Py_ssize_t elem_count = 1 @@ -115,6 +116,15 @@ cdef int _from_input_shape_strides( contig[0] = USM_ARRAY_C_CONTIGUOUS else: contig[0] = USM_ARRAY_F_CONTIGUOUS + if nd == 1: + contig[0] = USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS + else: + j = 0 + for i in range(nd): + if shape_arr[i] > 1: + j = j + 1 + if j < 2: + contig[0] = USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS min_disp[0] = 0 max_disp[0] = (elem_count - 1) strides_ptr[0] = (0) @@ -137,26 +147,42 @@ cdef int _from_input_shape_strides( min_disp[0] = min_shift max_disp[0] = max_shift if max_shift == min_shift + (elem_count - 1): + if elem_count == 1: + contig[0] = (USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS) + return 0 if nd == 1: if strides_arr[0] == 1: contig[0] = USM_ARRAY_C_CONTIGUOUS else: contig[0] = 0 return 0 - for i in range(0, nd - 1): - if all_incr: - all_incr = ( - (strides_arr[i] > 0) and - (strides_arr[i+1] > 0) and - (strides_arr[i] <= strides_arr[i + 1]) - ) - if all_decr: - all_decr = ( - (strides_arr[i] > 0) and - (strides_arr[i+1] > 0) and - (strides_arr[i] >= strides_arr[i + 1]) - ) - if all_incr: + i = 0 + while i < nd: + if shape_arr[i] == 1: + i = i + 1 + continue + j = i + 1 + while (j < nd and shape_arr[j] == 1): + j = j + 1 + if j < nd: + if all_incr: + all_incr = ( + (strides_arr[i] > 0) and + (strides_arr[j] > 0) and + (strides_arr[i] <= strides_arr[j]) + ) + if all_decr: + all_decr = ( + (strides_arr[i] > 0) and + (strides_arr[j] > 0) and + (strides_arr[i] >= strides_arr[j]) + ) + i = j + else: + break + if all_incr and all_decr: + contig[0] = (USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS) + elif all_incr: contig[0] = USM_ARRAY_F_CONTIGUOUS elif all_decr: contig[0] = USM_ARRAY_C_CONTIGUOUS diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 7a33e8d87c..b4d2ed7872 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -59,6 +59,16 @@ def test_allocate_usm_ndarray(shape, usm_type): assert X.shape == X.__sycl_usm_array_interface__["shape"] +def test_usm_ndarray_flags(): + assert dpt.usm_ndarray((5,)).flags == 3 + assert dpt.usm_ndarray((5, 2)).flags == 1 + assert dpt.usm_ndarray((5, 2), order="F").flags == 2 + assert dpt.usm_ndarray((5, 1, 2), order="F").flags == 2 + assert dpt.usm_ndarray((5, 1, 2), strides=(2, 0, 1)).flags == 1 + assert dpt.usm_ndarray((5, 1, 2), strides=(1, 0, 5)).flags == 2 + assert dpt.usm_ndarray((5, 1, 1), strides=(1, 0, 1)).flags == 3 + + @pytest.mark.parametrize( "dtype", [ @@ -703,11 +713,10 @@ def relaxed_strides_equal(st1, st2, sh): 5, ) X = dpt.usm_ndarray(sh_s, dtype="d") - expected_flags = X.flags X.shape = sh_f assert X.shape == sh_f assert relaxed_strides_equal(X.strides, cc_strides(sh_f), sh_f) - assert X.flags == expected_flags + assert X.flags & 1, "reshaped array expected to be C-contiguous" sh_s = ( 2, @@ -842,6 +851,10 @@ def test_reshape(): W = dpt.reshape(Z, (-1,), order="C") assert W.shape == (Z.size,) + X = dpt.usm_ndarray((1,)) + Y = dpt.reshape(X, X.shape) + assert Y.flags == X.flags + def test_transpose(): n, m = 2, 3 From f276b8d0fe325ffe2a0475a18a9963aa34d2b702 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 09:51:33 -0600 Subject: [PATCH 038/229] fixed typo in docstring --- dpctl/tensor/_dlpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 5cf799521d..741ca9d4c4 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -89,7 +89,7 @@ cdef extern from './include/dlpack/dlpack.h' nogil: def get_build_dlpack_version(): """ Returns the string value of DLPACK_VERSION from dlpack.h - `dpcl.tensor` was built with. + `dpctl.tensor` was built with. """ return str(DLPACK_VERSION) From 96636b20920a725b8798cffa912aa04ecf3456a7 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 13:30:17 -0600 Subject: [PATCH 039/229] Addressed docstrings/formatting PR feedback --- dpctl/tensor/_dlpack.pyx | 44 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 741ca9d4c4..2647904ef5 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -90,6 +90,10 @@ def get_build_dlpack_version(): """ Returns the string value of DLPACK_VERSION from dlpack.h `dpctl.tensor` was built with. + + Returns: + A string value of the version of DLPack used to build + `dpctl`. """ return str(DLPACK_VERSION) @@ -112,9 +116,26 @@ cdef void _managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: """ + to_dlpack_capsule(usm_ary) + Constructs named Python capsule object referencing instance of `DLManagerTensor` from :class:`dpctl.tensor.usm_ndarray` instance. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + Python a new capsule with name "dltensor" that contains + a pointer to `DLManagedTensor` struct. + Raises: + DLPackCreationError: when array can be represented as + DLPack tensor. This may happen when array was allocated + on a partitioned sycl device, or its USM allocation is + not bound to the platform default SYCL context. + MemoryError: when host allocation to needed for `DLManagedTensor` + did not succeed. + ValueError: when array elements data type could not be represented + in `DLManagedTensor`. """ cdef c_dpctl.SyclQueue ary_sycl_queue cdef c_dpctl.SyclDevice ary_sycl_device @@ -144,7 +165,6 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: "to_dlpack_capsule: DLPack can only export arrays allocated on " "non-partitioned SYCL devices." ) - # TODO: check that ary_sycl_context is the default context default_context = dpctl.SyclQueue(ary_sycl_device).sycl_context if not usm_ary.sycl_context == default_context: raise DLPackCreationError( @@ -152,7 +172,6 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: "allocations bound to a default platform SYCL context" ) - dlm_tensor = stdlib.malloc( sizeof(DLManagedTensor)) if dlm_tensor is NULL: @@ -244,9 +263,25 @@ cdef class _DLManagedTensorOwner: cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: """ + from_dlpack_capsule(caps) + Reconstructs instance of :class:`dpctl.tensor.usm_ndarray` from named Python capsule object referencing instance of `DLManagedTensor` without copy. The instance forms a view in the memory of the tensor. + + Args: + caps: Python capsule with name "dltensor" expected to reference + an instance of `DLManagedTensor` struct. + Returns: + Instance of :class:`dpctl.tensor.usm_ndarray` with a view into + memory of the tensor. Capsule is renamed to "used_dltensor" upon + success. + Raises: + TypeError: if argument is not a "dltensor" capsule. + ValueError: if argument is "used_dltensor" capsule, + if the USM pointer is not bound to the reconstructed + sycl context, or the DLPack's device_type is not supported + by dpctl. """ cdef DLManagedTensor *dlm_tensor = NULL cdef bytes usm_type @@ -355,7 +390,7 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: raise ValueError( "Can not import DLPack tensor with type code {}.".format( dlm_tensor.dl_tensor.dtype.code - ) + ) ) res_ary = usm_ndarray( py_shape, @@ -379,6 +414,9 @@ cpdef from_dlpack(array): object `obj` that implements `__dlpack__` protocol. The output array is always a zero-copy view of the input. + Args: + A Python object representing an array that supports `__dlpack__` + protocol. Raises: TypeError: if `obj` does not implement `__dlpack__` method. ValueError: if zero copy view can not be constructed because From 49293af463bfd2f0e085f073bd149c297b0de130 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 16:45:32 -0600 Subject: [PATCH 040/229] Applied consistent style for references --- dpctl/tensor/_dlpack.pyx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 2647904ef5..7d4ebfa0c3 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -72,18 +72,18 @@ cdef extern from './include/dlpack/dlpack.h' nogil: uint16_t lanes ctypedef struct DLTensor: - void* data + void *data DLDevice device int ndim DLDataType dtype - int64_t* shape - int64_t* strides + int64_t *shape + int64_t *strides uint64_t byte_offset ctypedef struct DLManagedTensor: DLTensor dl_tensor - void* manager_ctx - void (*deleter)(DLManagedTensor*) # noqa: E211 + void *manager_ctx + void (*deleter)(DLManagedTensor *) # noqa: E211 def get_build_dlpack_version(): @@ -141,15 +141,15 @@ cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: cdef c_dpctl.SyclDevice ary_sycl_device cdef DPCTLSyclDeviceRef pDRef = NULL cdef DLManagedTensor *dlm_tensor = NULL - cdef DLTensor* dl_tensor = NULL + cdef DLTensor *dl_tensor = NULL cdef int nd = usm_ary.get_ndim() - cdef char* data_ptr = usm_ary.get_data() + cdef char *data_ptr = usm_ary.get_data() cdef Py_ssize_t *shape_ptr = NULL cdef Py_ssize_t *strides_ptr = NULL cdef int64_t *shape_strides_ptr = NULL cdef int i = 0 cdef int device_id = -1 - cdef char* base_ptr = NULL + cdef char *base_ptr = NULL cdef Py_ssize_t element_offset = 0 cdef Py_ssize_t byte_offset = 0 @@ -291,7 +291,7 @@ cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: cdef Py_ssize_t offset_min = 0 cdef Py_ssize_t offset_max = 0 cdef int64_t stride_i - cdef char* mem_ptr = NULL + cdef char *mem_ptr = NULL cdef Py_ssize_t element_offset = 0 if not cpython.PyCapsule_IsValid(py_caps, 'dltensor'): From cc98753a2aa818e041625a6752e0df83ce7953f6 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 11:37:55 -0600 Subject: [PATCH 041/229] Added submodules to __all__ in dpctl/__init__.py Moved dpctl/tests/__init__.py and dpctl/tests/_helper.py into dpctl/tests/helper/ subfolder. Now dpctl's submodules can be automatically discovered with pkgutils ``` In [1]: import pkgutil, dpctl In [2]: [a.name for a in pkgutil.iter_modules(dpctl.__path__) if a.ispkg] Out[2]: ['memory', 'program', 'tensor', 'utils'] ``` These changes are being done to aid in automation of documentation generation. --- dpctl/__init__.py | 7 +++++++ dpctl/tests/{ => helper}/__init__.py | 16 +++++++++++++++- dpctl/tests/{ => helper}/_helper.py | 0 dpctl/tests/test_sycl_context.py | 3 +-- dpctl/tests/test_sycl_event.py | 3 +-- dpctl/tests/test_sycl_platform.py | 3 +-- dpctl/tests/test_sycl_queue.py | 3 +-- dpctl/tests/test_sycl_queue_manager.py | 3 +-- dpctl/tests/test_sycl_queue_memcpy.py | 3 +-- dpctl/tests/test_sycl_usm.py | 3 +-- 10 files changed, 29 insertions(+), 15 deletions(-) rename dpctl/tests/{ => helper}/__init__.py (73%) rename dpctl/tests/{ => helper}/_helper.py (100%) diff --git a/dpctl/__init__.py b/dpctl/__init__.py index efd93f3630..0052465b15 100644 --- a/dpctl/__init__.py +++ b/dpctl/__init__.py @@ -123,6 +123,13 @@ __all__ += [ "get_include", ] +# add submodules +__all__ += [ + "memory", + "program", + "tensor", + "utils", +] def get_include(): diff --git a/dpctl/tests/__init__.py b/dpctl/tests/helper/__init__.py similarity index 73% rename from dpctl/tests/__init__.py rename to dpctl/tests/helper/__init__.py index 2fc36c98cd..672505c384 100644 --- a/dpctl/tests/__init__.py +++ b/dpctl/tests/helper/__init__.py @@ -14,5 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Top-level module of all dpctl Python unit test cases. +"""Helper module for dpctl/tests """ + +from ._helper import ( + create_invalid_capsule, + has_cpu, + has_gpu, + has_sycl_platforms, +) + +__all__ = [ + "create_invalid_capsule", + "has_cpu", + "has_gpu", + "has_sycl_platforms", +] diff --git a/dpctl/tests/_helper.py b/dpctl/tests/helper/_helper.py similarity index 100% rename from dpctl/tests/_helper.py rename to dpctl/tests/helper/_helper.py diff --git a/dpctl/tests/test_sycl_context.py b/dpctl/tests/test_sycl_context.py index 9d09a6117c..99d72c14eb 100644 --- a/dpctl/tests/test_sycl_context.py +++ b/dpctl/tests/test_sycl_context.py @@ -18,11 +18,10 @@ """ import pytest +from helper import create_invalid_capsule import dpctl -from ._helper import create_invalid_capsule - list_of_valid_filter_selectors = [ "opencl", "opencl:gpu", diff --git a/dpctl/tests/test_sycl_event.py b/dpctl/tests/test_sycl_event.py index 4e76534bd8..24c4aa4b75 100644 --- a/dpctl/tests/test_sycl_event.py +++ b/dpctl/tests/test_sycl_event.py @@ -19,14 +19,13 @@ import numpy as np import pytest +from helper import create_invalid_capsule, has_cpu import dpctl import dpctl.memory as dpctl_mem import dpctl.program as dpctl_prog from dpctl import event_status_type as esty -from ._helper import create_invalid_capsule, has_cpu - def produce_event(profiling=False): oclSrc = " \ diff --git a/dpctl/tests/test_sycl_platform.py b/dpctl/tests/test_sycl_platform.py index 1333588d5e..3bc230cdb9 100644 --- a/dpctl/tests/test_sycl_platform.py +++ b/dpctl/tests/test_sycl_platform.py @@ -18,11 +18,10 @@ """ import pytest +from helper import has_sycl_platforms import dpctl -from ._helper import has_sycl_platforms - list_of_valid_filter_selectors = [ "opencl", "opencl:gpu", diff --git a/dpctl/tests/test_sycl_queue.py b/dpctl/tests/test_sycl_queue.py index bf35d76ae2..0f36c319b2 100644 --- a/dpctl/tests/test_sycl_queue.py +++ b/dpctl/tests/test_sycl_queue.py @@ -21,11 +21,10 @@ import sys import pytest +from helper import create_invalid_capsule import dpctl -from ._helper import create_invalid_capsule - list_of_standard_selectors = [ dpctl.select_accelerator_device, dpctl.select_cpu_device, diff --git a/dpctl/tests/test_sycl_queue_manager.py b/dpctl/tests/test_sycl_queue_manager.py index ae7c75cbbd..a863784af8 100644 --- a/dpctl/tests/test_sycl_queue_manager.py +++ b/dpctl/tests/test_sycl_queue_manager.py @@ -20,11 +20,10 @@ import contextlib import pytest +from helper import has_cpu, has_gpu, has_sycl_platforms import dpctl -from ._helper import has_cpu, has_gpu, has_sycl_platforms - @pytest.mark.skipif( not has_sycl_platforms(), reason="No SYCL platforms available" diff --git a/dpctl/tests/test_sycl_queue_memcpy.py b/dpctl/tests/test_sycl_queue_memcpy.py index f5cddc87b8..db1831b78c 100644 --- a/dpctl/tests/test_sycl_queue_memcpy.py +++ b/dpctl/tests/test_sycl_queue_memcpy.py @@ -18,12 +18,11 @@ """ import pytest +from helper import has_sycl_platforms import dpctl import dpctl.memory -from ._helper import has_sycl_platforms - def _create_memory(): nbytes = 1024 diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index cb1bfe30d4..0bb68edaa6 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -19,6 +19,7 @@ import numpy as np import pytest +from helper import has_cpu, has_gpu, has_sycl_platforms import dpctl from dpctl.memory import ( @@ -28,8 +29,6 @@ as_usm_memory, ) -from ._helper import has_cpu, has_gpu, has_sycl_platforms - class Dummy(MemoryUSMShared): """ From a8c91c7d93bdef0a7994b46455f59cc56bd878d1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 11:54:42 -0600 Subject: [PATCH 042/229] Merge pull request #697 from IntelPython/gh696-no-elems-nested-lists Fixed type inferencing for nest sequences with no elements --- dpctl/tensor/_ctors.py | 1 + dpctl/tests/test_tensor_asarray.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index 5126f40837..d0cce717b0 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -67,6 +67,7 @@ def _array_info_sequence(li): ) if dim is None: dim = tuple() + dt = float device = _host_set return (n,) + dim, dt, device diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py index 8c61fc9f13..d8c6f9d7ce 100644 --- a/dpctl/tests/test_tensor_asarray.py +++ b/dpctl/tests/test_tensor_asarray.py @@ -87,6 +87,11 @@ def test_asarray_from_sequence(): assert type(Y) is dpt.usm_ndarray assert Y.shape == (0,) + X = [[], []] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == (2, 0) + X = [True, False] Y = dpt.asarray(X, usm_type="device") assert type(Y) is dpt.usm_ndarray From f4d98e203a907c8c2292054dae789ae7bd76177e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 12:09:57 -0600 Subject: [PATCH 043/229] Add tests as package data for dpctl Since previously dpctl/tests/__init__.py existed, setuptools was discpvering dpctl/tests as subpackage which was thus included in the layout. With reorg, and removal of dpctl/tests/__init__.py, tests need to be specifically added as package data. include tests/helper/* in the layout dpctl/tests/conftest.py to make helper discoverable Based on ideas from https://stackoverflow.com/questions/33508060/create-and-import-helper-functions-in-tests-without-creating-packages-in-test-di --- dpctl/tests/conftest.py | 23 +++++++++++++++++++++++ setup.py | 1 + 2 files changed, 24 insertions(+) create mode 100644 dpctl/tests/conftest.py diff --git a/dpctl/tests/conftest.py b/dpctl/tests/conftest.py new file mode 100644 index 0000000000..6b85eea77c --- /dev/null +++ b/dpctl/tests/conftest.py @@ -0,0 +1,23 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Configures pytest to discover helper/ module +""" + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "helper")) diff --git a/setup.py b/setup.py index 928a8e0010..21b55e1151 100644 --- a/setup.py +++ b/setup.py @@ -463,6 +463,7 @@ def _get_cmdclass(): author="Intel Corporation", url="https://github.com/IntelPython/dpctl", packages=find_packages(include=["*"]), + package_data={"dpctl": ["tests/*", "tests/helper/*.py"]}, include_package_data=True, ext_modules=extensions(), zip_safe=False, From f31c30ccc4f53b14bc113926cb92635419be3c57 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Mon, 22 Nov 2021 21:07:09 +0300 Subject: [PATCH 044/229] Implemented error_handler func Rename dpctl_async_error_handler to dpctl_error_handlers DPCTL_VERBOSITY env handling is introduced. Recognized values: none (default), error, info Made sure to run coverage tests with DPCTL_VERBOSITY=error The TestDPCTLSyclQueueInterface.CheckPropertyHandling tests captured error stream in its test. Rename error_handler_callback *error_handler to prevert collisions --- dpctl/_backend.pxd | 6 +- ...error_handler.h => dpctl_error_handlers.h} | 19 ++++ .../source/dpctl_async_error_handler.cpp | 42 --------- .../helper/source/dpctl_error_handlers.cpp | 89 +++++++++++++++++++ .../include/dpctl_sycl_context_interface.h | 8 +- .../include/dpctl_sycl_queue_interface.h | 8 +- .../source/dpctl_sycl_context_interface.cpp | 12 ++- .../source/dpctl_sycl_queue_interface.cpp | 14 +-- libsyclinterface/tests/CMakeLists.txt | 2 +- 9 files changed, 132 insertions(+), 68 deletions(-) rename libsyclinterface/helper/include/{dpctl_async_error_handler.h => dpctl_error_handlers.h} (73%) delete mode 100644 libsyclinterface/helper/source/dpctl_async_error_handler.cpp create mode 100644 libsyclinterface/helper/source/dpctl_error_handlers.cpp diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index eefdd4da85..8a74222b06 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -281,11 +281,11 @@ cdef extern from "dpctl_sycl_platform_interface.h": cdef extern from "dpctl_sycl_context_interface.h": cdef DPCTLSyclContextRef DPCTLContext_Create( const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclContextRef DPCTLContext_CreateFromDevices( const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclContextRef DPCTLContext_Copy( const DPCTLSyclContextRef CRef) @@ -323,7 +323,7 @@ cdef extern from "dpctl_sycl_queue_interface.h": cdef DPCTLSyclQueueRef DPCTLQueue_Create( const DPCTLSyclContextRef CRef, const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclQueueRef DPCTLQueue_CreateForDevice( const DPCTLSyclDeviceRef dRef, diff --git a/libsyclinterface/helper/include/dpctl_async_error_handler.h b/libsyclinterface/helper/include/dpctl_error_handlers.h similarity index 73% rename from libsyclinterface/helper/include/dpctl_async_error_handler.h rename to libsyclinterface/helper/include/dpctl_error_handlers.h index f409f8646c..9bd93eec66 100644 --- a/libsyclinterface/helper/include/dpctl_async_error_handler.h +++ b/libsyclinterface/helper/include/dpctl_error_handlers.h @@ -44,3 +44,22 @@ class DPCTL_API DPCTL_AsyncErrorHandler void operator()(const cl::sycl::exception_list &exceptions); }; + +enum error_level : int +{ + none = 0, + error = 1, + warning = 2 +}; + +void error_handler(const std::exception &e, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type = error_level::error); + +void error_handler(const std::string &what, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type = error_level::warning); diff --git a/libsyclinterface/helper/source/dpctl_async_error_handler.cpp b/libsyclinterface/helper/source/dpctl_async_error_handler.cpp deleted file mode 100644 index 6b5d6db191..0000000000 --- a/libsyclinterface/helper/source/dpctl_async_error_handler.cpp +++ /dev/null @@ -1,42 +0,0 @@ -//===-- dpctl_async_error_handler.h - An async error handler -*-C++-*- ===// -// -// Data Parallel Control (dpctl) -// -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// A functor to use for passing an error handler callback function to sycl -/// context and queue contructors. -//===----------------------------------------------------------------------===// - -#include "dpctl_async_error_handler.h" - -void DPCTL_AsyncErrorHandler::operator()( - const cl::sycl::exception_list &exceptions) -{ - for (std::exception_ptr const &e : exceptions) { - try { - std::rethrow_exception(e); - } catch (cl::sycl::exception const &e) { - std::cerr << "Caught asynchronous SYCL exception:\n" - << e.what() << std::endl; - // FIXME: Change get_cl_code() to code() once DPCPP supports it. - auto err_code = e.get_cl_code(); - handler_(err_code); - } - } -} diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp new file mode 100644 index 0000000000..33beb7bb9f --- /dev/null +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -0,0 +1,89 @@ +//===-- dpctl_async_error_handler.h - An async error handler -*-C++-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// A functor to use for passing an error handler callback function to sycl +/// context and queue contructors. +//===----------------------------------------------------------------------===// + +#include "dpctl_error_handlers.h" +#include + +void DPCTL_AsyncErrorHandler::operator()( + const cl::sycl::exception_list &exceptions) +{ + for (std::exception_ptr const &e : exceptions) { + try { + std::rethrow_exception(e); + } catch (sycl::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); + auto err_code = e.get_cl_code(); + handler_(static_cast(err_code)); + } + } +} + +inline int get_requested_level(void) +{ + int requested_level = 0; + + const char *verbose = std::getenv("DPCTL_VERBOSITY"); + + if (verbose) { + if (!strncmp(verbose, "none", 4)) + requested_level = error_level::none; + else if (!strncmp(verbose, "error", 5)) + requested_level = error_level::error; + else if (!strncmp(verbose, "warning", 7)) + requested_level = error_level::warning; + } + + return requested_level; +} + +void error_handler(const std::exception &e, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type) +{ + int requested_level = get_requested_level(); + int error_level = static_cast(error_type); + + if (requested_level <= error_level) { + std::cerr << e.what() << " in " << func_name << " at " << file_name + << ":" << line_num << std::endl; + } +} + +void error_handler(const std::string &what, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type) +{ + int error_level = static_cast(error_type); + int requested_level = get_requested_level(); + + if (requested_level <= error_level) { + std::cerr << what << " In " << func_name << " at " << file_name << ":" + << line_num << std::endl; + } +} diff --git a/libsyclinterface/include/dpctl_sycl_context_interface.h b/libsyclinterface/include/dpctl_sycl_context_interface.h index f93125092d..77434c576a 100644 --- a/libsyclinterface/include/dpctl_sycl_context_interface.h +++ b/libsyclinterface/include/dpctl_sycl_context_interface.h @@ -46,7 +46,7 @@ DPCTL_C_EXTERN_C_BEGIN * optional async error handler and properties bit flags. * * @param DRef Opaque pointer to a SYCL device. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during context creation. Can be * NULL if no async_handler is needed. * @param properties An optional combination of bit flags to define @@ -58,7 +58,7 @@ DPCTL_C_EXTERN_C_BEGIN DPCTL_API __dpctl_give DPCTLSyclContextRef DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! @@ -67,7 +67,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, * * @param DVRef An opaque pointer to a std::vector of * DPCTLSyclDeviceRef opaque pointers. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during context creation. Can be * NULL if no async_handler is needed. * @param properties An optional combination of bit flags to define @@ -79,7 +79,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, DPCTL_API __dpctl_give DPCTLSyclContextRef DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! diff --git a/libsyclinterface/include/dpctl_sycl_queue_interface.h b/libsyclinterface/include/dpctl_sycl_queue_interface.h index 9f46a83e50..228604785e 100644 --- a/libsyclinterface/include/dpctl_sycl_queue_interface.h +++ b/libsyclinterface/include/dpctl_sycl_queue_interface.h @@ -47,7 +47,7 @@ DPCTL_C_EXTERN_C_BEGIN * * @param CRef An opaque pointer to a sycl::context. * @param DRef An opaque pointer to a sycl::device - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during queue creation. Can be * NULL if no async_handler is needed. * @param properties A combination of bit flags using the values defined @@ -62,7 +62,7 @@ DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, __dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! @@ -86,7 +86,7 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, * function begaves the same way as the SYCL constructor. * * @param dRef An opaque pointer to a ``sycl::device``. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during queue creation. Can be * NULL if no async_handler is needed. * @param properties A combination of bit flags using the values defined @@ -101,7 +101,7 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef dRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! diff --git a/libsyclinterface/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp index e26f6ed49c..6035819dd3 100644 --- a/libsyclinterface/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -25,7 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_context_interface.h" -#include "../helper/include/dpctl_async_error_handler.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" #include #include @@ -43,7 +43,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(std::vector, __dpctl_give DPCTLSyclContextRef DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int /**/) { DPCTLSyclContextRef CRef = nullptr; @@ -51,8 +51,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, if (!Device) return CRef; try { - CRef = - wrap(new context(*Device, DPCTL_AsyncErrorHandler(error_handler))); + CRef = wrap(new context(*Device, DPCTL_AsyncErrorHandler(handler))); } catch (const std::bad_alloc &ba) { std::cerr << ba.what() << '\n'; } catch (const runtime_error &re) { @@ -64,7 +63,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, __dpctl_give DPCTLSyclContextRef DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int /**/) { DPCTLSyclContextRef CRef = nullptr; @@ -79,8 +78,7 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, } try { - CRef = - wrap(new context(Devices, DPCTL_AsyncErrorHandler(error_handler))); + CRef = wrap(new context(Devices, DPCTL_AsyncErrorHandler(handler))); } catch (const std::bad_alloc &ba) { std::cerr << ba.what() << '\n'; } catch (const runtime_error &re) { diff --git a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp index 46bbd0d10f..4a7a7d4e27 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp @@ -25,7 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_interface.h" -#include "../helper/include/dpctl_async_error_handler.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_context_interface.h" #include "dpctl_sycl_device_interface.h" @@ -165,7 +165,7 @@ DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, __dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) { DPCTLSyclQueueRef q = nullptr; @@ -178,10 +178,10 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, } auto propList = create_property_list(properties); - if (propList && error_handler) { + if (propList && handler) { try { - auto Queue = new queue( - *ctx, *dev, DPCTL_AsyncErrorHandler(error_handler), *propList); + auto Queue = new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler), + *propList); q = wrap(Queue); } catch (std::bad_alloc const &ba) { std::cerr << ba.what() << '\n'; @@ -199,10 +199,10 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, std::cerr << re.what() << '\n'; } } - else if (error_handler) { + else if (handler) { try { auto Queue = - new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(error_handler)); + new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler)); q = wrap(Queue); } catch (std::bad_alloc const &ba) { std::cerr << ba.what() << '\n'; diff --git a/libsyclinterface/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt index 1000634794..1ecb4e847d 100644 --- a/libsyclinterface/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -59,7 +59,7 @@ if(DPCTL_GENERATE_COVERAGE) ) add_custom_target(llvm-cov COMMAND ${CMAKE_MAKE_PROGRAM} dpctl_c_api_tests - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests + COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=error ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests COMMAND ${LLVMProfdata_EXE} merge -sparse default.profraw From b1b89cd87e7349de9ba6291578fc883ff3bce64a Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Mon, 22 Nov 2021 22:28:48 +0300 Subject: [PATCH 045/229] Use error_handler in dpctl_sycl_platform_interface --- .../source/dpctl_sycl_platform_interface.cpp | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_platform_interface.cpp b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp index 03004433cb..1ae656b08f 100644 --- a/libsyclinterface/source/dpctl_sycl_platform_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_string_utils.hpp" #include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" @@ -50,15 +51,15 @@ DPCTLPlatform_Copy(__dpctl_keep const DPCTLSyclPlatformRef PRef) { auto Platform = unwrap(PRef); if (!Platform) { - std::cerr << "Cannot copy DPCTLSyclPlatformRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclPlatformRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedPlatform = new platform(*Platform); return wrap(CopiedPlatform); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -69,8 +70,8 @@ __dpctl_give DPCTLSyclPlatformRef DPCTLPlatform_Create() try { auto P = new platform(); PRef = wrap(P); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return PRef; } @@ -84,17 +85,15 @@ __dpctl_give DPCTLSyclPlatformRef DPCTLPlatform_CreateFromSelector( try { P = new platform(*DS); return wrap(P); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete P; - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Device selector pointer cannot be NULL\n"; + error_handler("Device selector pointer cannot be NULL.", __FILE__, + __func__, __LINE__); } return nullptr; @@ -115,7 +114,8 @@ DPCTLPlatform_GetBackend(__dpctl_keep const DPCTLSyclPlatformRef PRef) BTy = DPCTL_SyclBackendToDPCTLBackendType(P->get_backend()); } else { - std::cerr << "Backend cannot be looked up for a NULL platform\n"; + error_handler("Backend cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); } return BTy; } @@ -128,14 +128,14 @@ DPCTLPlatform_GetName(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto name = P->get_info(); return dpctl::helper::cstring_from_string(name); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Name cannot be looked up for a NULL platform\n"; + error_handler("Name cannot be looked up for a NULL platform.", __FILE__, + __func__, __LINE__); return nullptr; } } @@ -148,14 +148,14 @@ DPCTLPlatform_GetVendor(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto vendor = P->get_info(); return dpctl::helper::cstring_from_string(vendor); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Vendor cannot be looked up for a NULL platform\n"; + error_handler("Vendor cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -168,14 +168,14 @@ DPCTLPlatform_GetVersion(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto driver = P->get_info(); return dpctl::helper::cstring_from_string(driver); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Driver version cannot be looked up for a NULL platform\n"; + error_handler("Driver version cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -189,7 +189,8 @@ __dpctl_give DPCTLPlatformVectorRef DPCTLPlatform_GetPlatforms() try { Platforms = new std::vector(); Platforms->reserve(platforms.size()); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } From 504d798b86f70fe295f9552814a1d54acabba6be Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 25 Nov 2021 12:37:06 +0300 Subject: [PATCH 046/229] Use eror_handler in dpctl_sycl_context_interface --- .../source/dpctl_sycl_context_interface.cpp | 69 ++++++++++--------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp index 6035819dd3..d6a74833dd 100644 --- a/libsyclinterface/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -48,14 +48,16 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, { DPCTLSyclContextRef CRef = nullptr; auto Device = unwrap(DRef); - if (!Device) - return CRef; + if (!Device) { + error_handler("Cannot create device from DPCTLSyclDeviceRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); + return nullptr; + } try { CRef = wrap(new context(*Device, DPCTL_AsyncErrorHandler(handler))); - } catch (const std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - } catch (const runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return CRef; @@ -69,8 +71,12 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, DPCTLSyclContextRef CRef = nullptr; std::vector Devices; auto DeviceRefs = unwrap(DVRef); - if (!DeviceRefs) + if (!DeviceRefs) { + error_handler("Cannot create device reference from DPCTLDeviceVectorRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return CRef; + } Devices.reserve(DeviceRefs->size()); for (auto const &DRef : *DeviceRefs) { @@ -79,10 +85,8 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, try { CRef = wrap(new context(Devices, DPCTL_AsyncErrorHandler(handler))); - } catch (const std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - } catch (const runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return CRef; @@ -91,9 +95,11 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, bool DPCTLContext_AreEq(__dpctl_keep const DPCTLSyclContextRef CtxRef1, __dpctl_keep const DPCTLSyclContextRef CtxRef2) { - if (!(CtxRef1 && CtxRef2)) - // \todo handle error + if (!(CtxRef1 && CtxRef2)) { + error_handler("DPCTLSyclContextRefs are nullptr.", __FILE__, __func__, + __LINE__); return false; + } return (*unwrap(CtxRef1) == *unwrap(CtxRef2)); } @@ -102,15 +108,15 @@ DPCTLContext_Copy(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Cannot copy DPCTLSyclContextRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclContextRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedContext = new context(*Context); return wrap(CopiedContext); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -120,16 +126,17 @@ DPCTLContext_GetDevices(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Can not retrieve devices from DPCTLSyclContextRef as " - "input is a nullptr\n"; + error_handler("Cannot retrieve devices from DPCTLSyclContextRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } std::vector *DevicesVectorPtr = nullptr; try { DevicesVectorPtr = new std::vector(); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + delete DevicesVectorPtr; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -139,15 +146,9 @@ DPCTLContext_GetDevices(__dpctl_keep const DPCTLSyclContextRef CRef) DevicesVectorPtr->emplace_back(wrap(new device(Dev))); } return wrap(DevicesVectorPtr); - } catch (std::bad_alloc const &ba) { - delete DevicesVectorPtr; - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (const runtime_error &re) { + } catch (std::exception const &e) { delete DevicesVectorPtr; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -156,8 +157,9 @@ size_t DPCTLContext_DeviceCount(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Can not retrieve devices from DPCTLSyclContextRef as " - "input is a nullptr\n"; + error_handler("Cannot retrieve devices from DPCTLSyclContextRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); return 0; } const auto Devices = Context->get_devices(); @@ -205,8 +207,7 @@ size_t DPCTLContext_Hash(__dpctl_keep const DPCTLSyclContextRef CtxRef) return hash_fn(*C); } else { - std::cerr << "Argument CtxRef is null" - << "/n"; + error_handler("Argument CtxRef is null.", __FILE__, __func__, __LINE__); return 0; } } From 380c54db20c22b3416cbe7cbfee0dcf54273b15d Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 25 Nov 2021 17:59:16 +0300 Subject: [PATCH 047/229] Use error_handler in dpctl_sycl_device_interface --- .../source/dpctl_sycl_device_interface.cpp | 213 ++++++------------ 1 file changed, 75 insertions(+), 138 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_device_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_interface.cpp index dbe42e764b..804998cf19 100644 --- a/libsyclinterface/source/dpctl_sycl_device_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_interface.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_string_utils.hpp" #include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" @@ -52,15 +53,15 @@ DPCTLDevice_Copy(__dpctl_keep const DPCTLSyclDeviceRef DRef) { auto Device = unwrap(DRef); if (!Device) { - std::cerr << "Cannot copy DPCTLSyclDeviceRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclDeviceRef as input is a nullptr", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedDevice = new device(*Device); return wrap(CopiedDevice); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -70,13 +71,8 @@ __dpctl_give DPCTLSyclDeviceRef DPCTLDevice_Create() try { auto Device = new device(); return wrap(Device); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -85,19 +81,17 @@ __dpctl_give DPCTLSyclDeviceRef DPCTLDevice_CreateFromSelector( __dpctl_keep const DPCTLSyclDeviceSelectorRef DSRef) { auto Selector = unwrap(DSRef); - if (!Selector) - // \todo : Log error + if (!Selector) { + error_handler("Cannot difine device selector for DPCTLSyclDeviceRef " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; + } try { auto Device = new device(*Selector); return wrap(Device); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -116,9 +110,8 @@ DPCTLDevice_GetDeviceType(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto SyclDTy = D->get_info(); DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType(SyclDTy); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return DTy; @@ -180,9 +173,8 @@ DPCTLDevice_GetMaxComputeUnits(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { nComputeUnits = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return nComputeUnits; @@ -196,9 +188,8 @@ DPCTLDevice_GetGlobalMemSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { GlobalMemSize = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return GlobalMemSize; @@ -211,9 +202,8 @@ uint64_t DPCTLDevice_GetLocalMemSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { LocalMemSize = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return LocalMemSize; @@ -228,9 +218,8 @@ DPCTLDevice_GetMaxWorkItemDims(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { maxWorkItemDims = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return maxWorkItemDims; @@ -248,12 +237,8 @@ DPCTLDevice_GetMaxWorkItemSizes(__dpctl_keep const DPCTLSyclDeviceRef DRef) for (auto i = 0ul; i < 3; ++i) { sizes[i] = id_sizes[i]; } - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return sizes; @@ -267,9 +252,8 @@ DPCTLDevice_GetMaxWorkGroupSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { max_wg_size = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return max_wg_size; @@ -283,9 +267,8 @@ DPCTLDevice_GetMaxNumSubGroups(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { max_nsubgroups = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return max_nsubgroups; @@ -299,8 +282,8 @@ DPCTLDevice_GetPlatform(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { PRef = wrap(new platform(D->get_platform())); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return PRef; @@ -315,9 +298,8 @@ DPCTLDevice_GetName(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto name = D->get_info(); cstr_name = dpctl::helper::cstring_from_string(name); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_name; @@ -332,9 +314,8 @@ DPCTLDevice_GetVendor(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto vendor = D->get_info(); cstr_vendor = dpctl::helper::cstring_from_string(vendor); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_vendor; @@ -349,9 +330,8 @@ DPCTLDevice_GetDriverVersion(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto driver = D->get_info(); cstr_driver = dpctl::helper::cstring_from_string(driver); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_driver; @@ -364,9 +344,8 @@ bool DPCTLDevice_IsHostUnifiedMemory(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { ret = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return ret; @@ -391,9 +370,8 @@ bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, if (D) { try { hasAspect = D->has(DPCTL_DPCTLAspectTypeToSyclAspect(AT)); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return hasAspect; @@ -407,8 +385,8 @@ bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, if (D) { \ try { \ result = D->get_info(); \ - } catch (runtime_error const &re) { \ - std::cerr << re.what() << '\n'; \ + } catch (std::exception const &e) { \ + error_handler(e, __FILE__, __func__, __LINE__); \ } \ } \ return result; \ @@ -431,9 +409,8 @@ bool DPCTLDevice_GetSubGroupIndependentForwardProgress( try { SubGroupProgress = D->get_info< info::device::sub_group_independent_forward_progress>(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return SubGroupProgress; @@ -448,9 +425,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthChar( try { vector_width_char = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_char; @@ -465,9 +441,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthShort( try { vector_width_short = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_short; @@ -482,9 +457,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthInt( try { vector_width_int = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_int; @@ -499,9 +473,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthLong( try { vector_width_long = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_long; @@ -516,9 +489,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthFloat( try { vector_width_float = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_float; @@ -533,9 +505,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthDouble( try { vector_width_double = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_double; @@ -550,9 +521,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthHalf( try { vector_width_half = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_half; @@ -566,12 +536,8 @@ DPCTLDevice_GetParentDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto parent_D = D->get_info(); return wrap(new device(parent_D)); - } catch (invalid_object_error const &ioe) { - // not a sub device - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -586,8 +552,8 @@ DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, std::vector *Devices = nullptr; if (DRef) { if (count == 0) { - std::cerr << "Can not create sub-devices with zero compute units" - << '\n'; + error_handler("Cannot create sub-devices with zero compute units", + __FILE__, __func__, __LINE__); return nullptr; } auto D = unwrap(DRef); @@ -598,18 +564,9 @@ DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (feature_not_supported const &fnse) { - delete Devices; - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -626,8 +583,8 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, vcounts.assign(counts, counts + ncounts); size_t min_elem = *std::min_element(vcounts.begin(), vcounts.end()); if (min_elem == 0) { - std::cerr << "Can not create sub-devices with zero compute units" - << '\n'; + error_handler("Cannot create sub-devices with zero compute units", + __FILE__, __func__, __LINE__); return nullptr; } if (DRef) { @@ -636,12 +593,8 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, try { subDevices = D->create_sub_devices< info::partition_property::partition_by_counts>(vcounts); - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -649,14 +602,9 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -679,18 +627,9 @@ __dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (feature_not_supported const &fnse) { - delete Devices; - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -705,9 +644,7 @@ size_t DPCTLDevice_Hash(__dpctl_keep const DPCTLSyclDeviceRef DRef) return hash_fn(*D); } else { - // todo: log error - std::cerr << "Argument DRef is null" - << "/n"; + error_handler("Argument DRef is null", __FILE__, __func__, __LINE__); return 0; } } From 033dac0ef0c561e4a376019883baedc72776824c Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 25 Nov 2021 23:28:46 +0300 Subject: [PATCH 048/229] Use error_handler in dpctl_sycl_device_manager --- .../source/dpctl_sycl_device_manager.cpp | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp index 67446d6350..06d90c039c 100644 --- a/libsyclinterface/source/dpctl_sycl_device_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_manager.cpp @@ -24,6 +24,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_manager.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_string_utils.hpp" #include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" @@ -134,8 +135,9 @@ struct DeviceCacheBuilder auto entry = cache_l.emplace(D, Ctx); if (!entry.second) { - std::cerr << "Fatal Error during device cache " - "construction.\n"; + error_handler("Fatal Error during device cache " + "construction.", + __FILE__, __func__, __LINE__); std::terminate(); } } @@ -160,9 +162,12 @@ DPCTLDeviceMgr_GetCachedContext(__dpctl_keep const DPCTLSyclDeviceRef DRef) DPCTLSyclContextRef CRef = nullptr; auto Device = unwrap(DRef); - if (!Device) + if (!Device) { + error_handler("Cannot create device from DPCTLSyclDeviceRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return CRef; - + } auto &cache = DeviceCacheBuilder::getDeviceCache(); auto entry = cache.find(*Device); if (entry != cache.end()) { @@ -170,14 +175,15 @@ DPCTLDeviceMgr_GetCachedContext(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { ContextPtr = new context(entry->second); CRef = wrap(ContextPtr); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); delete ContextPtr; CRef = nullptr; } } else { - std::cerr << "No cached default context for device" << std::endl; + error_handler("No cached default context for device.", __FILE__, + __func__, __LINE__); } return CRef; } @@ -191,8 +197,9 @@ DPCTLDeviceMgr_GetDevices(int device_identifier) try { Devices = new std::vector(); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Devices; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -226,9 +233,8 @@ DPCTLDeviceMgr_GetDeviceInfoStr(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto infostr = get_device_info_str(*D); cstr_info = dpctl::helper::cstring_from_string(infostr); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_info; @@ -303,7 +309,8 @@ void DPCTLDeviceMgr_PrintDeviceInfo(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (Device) std::cout << get_device_info_str(*Device); else - std::cout << "Device is not valid (NULL). Cannot print device info.\n"; + error_handler("Device is not valid (NULL). Cannot print device info.", + __FILE__, __func__, __LINE__); } int64_t DPCTLDeviceMgr_GetRelativeId(__dpctl_keep const DPCTLSyclDeviceRef DRef) From 0d10f0c3c92c8a5f5061d410acd2f0f0f37dcf95 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 00:58:20 +0300 Subject: [PATCH 049/229] Use error_handler in dpctl_sycl_device_selector_interface --- .../dpctl_sycl_device_selector_interface.cpp | 43 ++++++------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp index 1193e11cee..8793775806 100644 --- a/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp @@ -24,6 +24,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_selector_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" #include /* SYCL headers */ @@ -42,11 +43,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLAcceleratorSelector_Create() try { auto Selector = new accelerator_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -56,11 +54,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLDefaultSelector_Create() try { auto Selector = new default_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -70,11 +65,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLCPUSelector_Create() try { auto Selector = new cpu_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -90,11 +82,8 @@ DPCTLFilterSelector_Create(__dpctl_keep const char *filter_str) try { auto Selector = new filter_selector_t(filter_str); return wrap(Selector); - } catch (std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error &re) { - std::cerr << "Device: " << filter_str << " " << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -104,11 +93,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLGPUSelector_Create() try { auto Selector = new gpu_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -118,11 +104,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLHostSelector_Create() try { auto Selector = new host_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } From 0a2c1dca1606a7807c267831c7bcb96bb5a6721b Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 01:17:47 +0300 Subject: [PATCH 050/229] Use error_handler in dpctl_sycl_event_interface --- .../source/dpctl_sycl_event_interface.cpp | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_event_interface.cpp b/libsyclinterface/source/dpctl_sycl_event_interface.cpp index 5d36ef3dee..2ba4b28011 100644 --- a/libsyclinterface/source/dpctl_sycl_event_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_event_interface.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_event_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include /* SYCL headers */ @@ -49,23 +50,23 @@ __dpctl_give DPCTLSyclEventRef DPCTLEvent_Create() try { auto E = new event(); ERef = wrap(E); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return ERef; } void DPCTLEvent_Wait(__dpctl_keep DPCTLSyclEventRef ERef) { - // \todo How to handle errors? E.g. when ERef is null or not a valid event. if (ERef) { auto SyclEvent = unwrap(ERef); if (SyclEvent) SyclEvent->wait(); } else { - std::cerr << "Cannot wait for the event. DPCTLSyclEventRef as input is " - "a nullptr\n"; + error_handler("Cannot wait for the event. DPCTLSyclEventRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); } } @@ -77,9 +78,9 @@ void DPCTLEvent_WaitAndThrow(__dpctl_keep DPCTLSyclEventRef ERef) SyclEvent->wait_and_throw(); } else { - std::cerr << "Cannot wait_and_throw for the event. DPCTLSyclEventRef " - "as input is " - "a nullptr\n"; + error_handler("Cannot wait_and_throw for the event. DPCTLSyclEventRef " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); } } @@ -93,15 +94,15 @@ DPCTLEvent_Copy(__dpctl_keep DPCTLSyclEventRef ERef) { auto SyclEvent = unwrap(ERef); if (!SyclEvent) { - std::cerr << "Cannot copy DPCTLSyclEventRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclEventRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedSyclEvent = new event(*SyclEvent); return wrap(CopiedSyclEvent); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -114,7 +115,8 @@ DPCTLSyclBackendType DPCTLEvent_GetBackend(__dpctl_keep DPCTLSyclEventRef ERef) BTy = DPCTL_SyclBackendToDPCTLBackendType(E->get_backend()); } else { - std::cerr << "Backend cannot be looked up for a NULL event\n"; + error_handler("Backend cannot be looked up for a NULL event.", __FILE__, + __func__, __LINE__); } return BTy; } @@ -130,9 +132,8 @@ DPCTLEvent_GetCommandExecutionStatus(__dpctl_keep DPCTLSyclEventRef ERef) auto SyclESTy = E->get_info(); ESTy = DPCTL_SyclEventStatusToDPCTLEventStatusType(SyclESTy); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return ESTy; @@ -147,9 +148,8 @@ uint64_t DPCTLEvent_GetProfilingInfoSubmit(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoSubmit = E->get_profiling_info< sycl::info::event_profiling::command_submit>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoSubmit; @@ -164,9 +164,8 @@ uint64_t DPCTLEvent_GetProfilingInfoStart(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoStart = E->get_profiling_info< sycl::info::event_profiling::command_start>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoStart; @@ -181,9 +180,8 @@ uint64_t DPCTLEvent_GetProfilingInfoEnd(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoEnd = E->get_profiling_info< sycl::info::event_profiling::command_end>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoEnd; @@ -194,15 +192,15 @@ DPCTLEvent_GetWaitList(__dpctl_keep DPCTLSyclEventRef ERef) { auto E = unwrap(ERef); if (!E) { - std::cerr << "Cannot get wait list as input is a nullptr\n"; + error_handler("Cannot get wait list as input is a nullptr.", __FILE__, + __func__, __LINE__); return nullptr; } std::vector *EventsVectorPtr = nullptr; try { EventsVectorPtr = new std::vector(); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -212,15 +210,9 @@ DPCTLEvent_GetWaitList(__dpctl_keep DPCTLSyclEventRef ERef) EventsVectorPtr->emplace_back(wrap(new event(Ev))); } return wrap(EventsVectorPtr); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete EventsVectorPtr; - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (const runtime_error &re) { - delete EventsVectorPtr; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } From 36e0aecb8e99f6b6bf1b62b2429d8ce484135834 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 01:40:49 +0300 Subject: [PATCH 051/229] Use error_handler in dpctl_sycl_kernel_interface --- libsyclinterface/source/dpctl_sycl_kernel_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp index b25d73ada0..a1a7e38e39 100644 --- a/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_kernel_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_string_utils.hpp" #include "Support/CBindingWrapping.h" #include /* Sycl headers */ @@ -42,7 +43,9 @@ __dpctl_give const char * DPCTLKernel_GetFunctionName(__dpctl_keep const DPCTLSyclKernelRef Kernel) { if (!Kernel) { - // \todo record error + error_handler("Cannot get the number of arguments " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } @@ -56,7 +59,9 @@ DPCTLKernel_GetFunctionName(__dpctl_keep const DPCTLSyclKernelRef Kernel) size_t DPCTLKernel_GetNumArgs(__dpctl_keep const DPCTLSyclKernelRef Kernel) { if (!Kernel) { - // \todo record error + error_handler("Cannot get the number of arguments from " + "DPCTLSyclKernelRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return -1; } From 90aab6ebcbc76c2f864b076a826b687f82acbe7c Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 01:58:44 +0300 Subject: [PATCH 052/229] Use error_handler in dpctl_sycl_platform_manager --- libsyclinterface/source/dpctl_sycl_platform_manager.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_platform_manager.cpp b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp index 11a4b65cf2..679b0c6350 100644 --- a/libsyclinterface/source/dpctl_sycl_platform_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_manager.h" +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_platform_interface.h" @@ -45,8 +46,9 @@ void platform_print_info_impl(const platform &p, size_t verbosity) std::stringstream ss; if (verbosity > 2) { - std::cerr << "Illegal verbosity level. Accepted values are 0, 1, or 2. " - "Defaulting to verbosity level 0.\n"; + error_handler("Illegal verbosity level. Accepted values are 0, 1, or 2." + "Defaulting to verbosity level 0.", + __FILE__, __func__, __LINE__); verbosity = 0; } @@ -112,6 +114,7 @@ void DPCTLPlatformMgr_PrintInfo(__dpctl_keep const DPCTLSyclPlatformRef PRef, platform_print_info_impl(*p, verbosity); } else { - std::cout << "Platform reference is NULL.\n"; + error_handler("Platform reference is NULL.", __FILE__, __func__, + __LINE__); } } From cfa5a05206ae65784b3f355252742829d3f88a42 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 02:55:23 +0300 Subject: [PATCH 053/229] Use error_handler in dpctl_sycl_usm_interface --- .../source/dpctl_sycl_usm_interface.cpp | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_usm_interface.cpp b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp index be1becb1f1..2347898c36 100644 --- a/libsyclinterface/source/dpctl_sycl_usm_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_usm_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_interface.h" #include /* SYCL headers */ @@ -45,15 +46,15 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_shared(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = malloc_shared(size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -64,15 +65,15 @@ DPCTLaligned_alloc_shared(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = aligned_alloc_shared(alignment, size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -81,7 +82,7 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_host(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } // SYCL 2020 spec: for devices without aspect::usm_host_allocations: @@ -97,7 +98,7 @@ DPCTLaligned_alloc_host(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } // SYCL 2020 spec: for devices without aspect::usm_host_allocations: @@ -111,15 +112,15 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_device(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = malloc_device(size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -130,15 +131,15 @@ DPCTLaligned_alloc_device(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = aligned_alloc_device(alignment, size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -147,11 +148,12 @@ void DPCTLfree_with_queue(__dpctl_take DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return; } if (!MRef) { - std::cerr << "Input MRef is nullptr, nothing to free\n"; + error_handler("Input MRef is nullptr, nothing to free.", __FILE__, + __func__, __LINE__); return; } auto Ptr = unwrap(MRef); @@ -163,11 +165,12 @@ void DPCTLfree_with_context(__dpctl_take DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return; } if (!MRef) { - std::cerr << "Input MRef is nullptr, nothing to free\n"; + error_handler("Input MRef is nullptr, nothing to free.", __FILE__, + __func__, __LINE__); return; } auto Ptr = unwrap(MRef); @@ -179,11 +182,11 @@ const char *DPCTLUSM_GetPointerType(__dpctl_keep const DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return "unknown"; } if (!MRef) { - std::cerr << "Input MRef is nullptr\n"; + error_handler("Input MRef is nullptr.", __FILE__, __func__, __LINE__); return "unknown"; } auto Ptr = unwrap(MRef); @@ -207,11 +210,11 @@ DPCTLUSM_GetPointerDevice(__dpctl_keep const DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } if (!MRef) { - std::cerr << "Input MRef is nullptr\n"; + error_handler("Input MRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } From 8562e6775d90ed334c26f4f036dc82c41ed5be47 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 03:01:49 +0300 Subject: [PATCH 054/229] Use error_handler in dpctl_vector_templ --- libsyclinterface/source/dpctl_vector_templ.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libsyclinterface/source/dpctl_vector_templ.cpp b/libsyclinterface/source/dpctl_vector_templ.cpp index 36e1bd4e65..a2f9e8def9 100644 --- a/libsyclinterface/source/dpctl_vector_templ.cpp +++ b/libsyclinterface/source/dpctl_vector_templ.cpp @@ -23,6 +23,7 @@ /// the wrapper functions for vector operations. /// //===----------------------------------------------------------------------===// +#include "../helper/include/dpctl_error_handlers.h" #include "../helper/include/dpctl_vector_macros.h" #include "Support/MemOwnershipAttrs.h" #include @@ -44,8 +45,9 @@ __dpctl_give VECTOR(EL) FN(EL, Create)() try { Vec = new std::vector(); return wrap(Vec); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Vec; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -68,8 +70,9 @@ __dpctl_give VECTOR(EL) wrap(new std::remove_pointer::type(*Ref))); } return wrap(Vec); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Vec; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -133,8 +136,8 @@ SYCLREF(EL) FN(EL, GetAt)(__dpctl_keep VECTOR(EL) VRef, size_t index) SYCLREF(EL) ret; try { ret = Vec->at(index); - } catch (std::out_of_range const &oor) { - std::cerr << oor.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } auto Ref = unwrap(ret); @@ -142,10 +145,9 @@ SYCLREF(EL) FN(EL, GetAt)(__dpctl_keep VECTOR(EL) VRef, size_t index) try { elPtr = new std::remove_pointer::type(*Ref); copy = wrap(elPtr); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete elPtr; - // \todo log error - std::cerr << ba.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } From 8bdec8c635b262ab4ecbb861af20e0a6d66bfb44 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 12:50:47 +0300 Subject: [PATCH 055/229] Use eror_handler in dpctl_sycl_program_interface --- .../source/dpctl_sycl_program_interface.cpp | 91 +++++++++---------- 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index fc2f443beb..7e24d3fd34 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -30,11 +30,13 @@ #endif #include "dpctl_sycl_program_interface.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Config/dpctl_config.h" #include "Support/CBindingWrapping.h" #include /* OpenCL headers */ #include /* Sycl headers */ #include +#include #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION #include "../helper/include/dpctl_dynamic_lib_helper.h" @@ -84,10 +86,11 @@ createOpenCLInterOpProgram(const context &SyclCtx, auto CLCtx = get_native(SyclCtx); auto CLProgram = clCreateProgramWithIL(CLCtx, IL, length, &err); if (err) { - // \todo: record the error string and any other information. - std::cerr << "OpenCL program could not be created from the SPIR-V " - "binary. OpenCL Error " - << err << ".\n"; + std::stringstream ss; + ss << "OpenCL program could not be created from the SPIR-V " + "binary. OpenCL Error " + << err << "."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); return nullptr; } auto SyclDevices = SyclCtx.get_devices(); @@ -104,9 +107,9 @@ createOpenCLInterOpProgram(const context &SyclCtx, delete[] CLDevices; if (err) { - // \todo: record the error string and any other information. - std::cerr << "OpenCL program could not be built. OpenCL Error " << err - << ".\n"; + std::stringstream ss; + ss << "OpenCL program could not be built. OpenCL Error " << err << "."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); return nullptr; } @@ -114,9 +117,8 @@ createOpenCLInterOpProgram(const context &SyclCtx, try { auto SyclProgram = new program(SyclCtx, CLProgram); return wrap(SyclProgram); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -127,9 +129,9 @@ zeModuleCreateFT getZeModuleCreateFn() { static dpctl::DynamicLibHelper zeLib(zeLoaderName, libLoadFlags); if (!zeLib.opened()) { - // TODO: handle error - std::cerr << "The level zero loader dynamic library could not " - "be opened.\n"; + error_handler("The level zero loader dynamic library could not " + "be opened.", + __FILE__, __func__, __LINE__); return nullptr; } static auto stZeModuleCreateF = @@ -147,8 +149,8 @@ createLevelZeroInterOpProgram(const context &SyclCtx, auto ZeCtx = get_native(SyclCtx); auto SyclDevices = SyclCtx.get_devices(); if (SyclDevices.size() > 1) { - std::cerr << "Level zero program can be created for only one device.\n"; - // TODO: handle error + error_handler("Level zero program can be created for only one device.", + __FILE__, __func__, __LINE__); return nullptr; } @@ -172,15 +174,16 @@ createLevelZeroInterOpProgram(const context &SyclCtx, auto stZeModuleCreateF = getZeModuleCreateFn(); if (!stZeModuleCreateF) { - std::cerr << "ZeModuleCreateFn is invalid.\n"; + error_handler("ZeModuleCreateFn is invalid.", __FILE__, __func__, + __LINE__); return nullptr; } auto ret = stZeModuleCreateF(ZeCtx, ZeDevice, &ZeModuleDesc, &ZeModule, nullptr); if (ret != ZE_RESULT_SUCCESS) { - // TODO: handle error - std::cerr << "ZeModule creation failed.\n"; + error_handler("ZeModule creation failed.", __FILE__, __func__, + __LINE__); return nullptr; } @@ -189,9 +192,8 @@ createLevelZeroInterOpProgram(const context &SyclCtx, auto ZeProgram = new program(sycl::level_zero::make_program( SyclCtx, reinterpret_cast(ZeModule))); return wrap(ZeProgram); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -208,9 +210,9 @@ DPCTLProgram_CreateFromSpirv(__dpctl_keep const DPCTLSyclContextRef CtxRef, DPCTLSyclProgramRef Pref = nullptr; context *SyclCtx = nullptr; if (!CtxRef) { - // \todo handle error - std::cerr << "Cannot create program from SPIR-V as the supplied SYCL " - "context is NULL.\n"; + error_handler("Cannot create program from SPIR-V as the supplied SYCL " + "context is NULL.", + __FILE__, __func__, __LINE__); return Pref; } SyclCtx = unwrap(CtxRef); @@ -241,12 +243,12 @@ DPCTLProgram_CreateFromOCLSource(__dpctl_keep const DPCTLSyclContextRef Ctx, program *SyclProgram = nullptr; if (!Ctx) { - // \todo handle error + error_handler("Input Ctx is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } if (!Source) { - // \todo handle error message + error_handler("Input Source is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } @@ -265,29 +267,20 @@ DPCTLProgram_CreateFromOCLSource(__dpctl_keep const DPCTLSyclContextRef Ctx, try { SyclProgram->build_with_source(source, compileOpts); return wrap(SyclProgram); - } catch (compile_program_error &e) { - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { delete SyclProgram; - // \todo record error - return nullptr; - } catch (feature_not_supported &e) { - std::cerr << e.what() << '\n'; - delete SyclProgram; - // \todo record error - return nullptr; - } catch (runtime_error &e) { - std::cerr << e.what() << '\n'; - delete SyclProgram; - // \todo record error + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } break; case backend::level_zero: - std::cerr << "CreateFromSource is not supported in Level Zero.\n"; + error_handler("CreateFromSource is not supported in Level Zero.", + __FILE__, __func__, __LINE__); delete SyclProgram; return nullptr; default: - std::cerr << "CreateFromSource is not supported in unknown backend.\n"; + error_handler("CreateFromSource is not supported in unknown backend.", + __FILE__, __func__, __LINE__); delete SyclProgram; return nullptr; } @@ -298,21 +291,21 @@ DPCTLProgram_GetKernel(__dpctl_keep DPCTLSyclProgramRef PRef, __dpctl_keep const char *KernelName) { if (!PRef) { - // \todo record error + error_handler("Input PRef is nullptr", __FILE__, __func__, __LINE__); return nullptr; } auto SyclProgram = unwrap(PRef); if (!KernelName) { - // \todo record error + error_handler("Input KernelName is nullptr", __FILE__, __func__, + __LINE__); return nullptr; } std::string name = KernelName; try { auto SyclKernel = new kernel(SyclProgram->get_kernel(name)); return wrap(SyclKernel); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -321,15 +314,15 @@ bool DPCTLProgram_HasKernel(__dpctl_keep DPCTLSyclProgramRef PRef, __dpctl_keep const char *KernelName) { if (!PRef) { - // \todo handle error + error_handler("Input PRef is nullptr", __FILE__, __func__, __LINE__); return false; } auto SyclProgram = unwrap(PRef); try { return SyclProgram->has_kernel(KernelName); - } catch (invalid_object_error &e) { - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return false; } } From 4cd91648e48665d9f8536c54cd35a9c3699684b4 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 26 Nov 2021 18:01:55 +0300 Subject: [PATCH 056/229] Use eror_handler in dpctl_sycl_queue_interface --- .../source/dpctl_sycl_queue_interface.cpp | 145 ++++++++---------- 1 file changed, 60 insertions(+), 85 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp index 4a7a7d4e27..152bec562d 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp @@ -108,9 +108,9 @@ bool set_kernel_arg(handler &cgh, cgh.set_arg(idx, Arg); break; default: - // \todo handle errors arg_set = false; - std::cerr << "Kernel argument could not be created.\n"; + error_handler("Kernel argument could not be created.", __FILE__, + __func__, __LINE__); break; } return arg_set; @@ -140,10 +140,10 @@ std::unique_ptr create_property_list(int properties) } if (_prop) { - // todo: log error - std::cerr << "Invalid queue property argument (" << std::hex - << properties << "), interpreted as (" << (properties ^ _prop) - << ")" << '\n'; + std::stringstream ss; + ss << "Invalid queue property argument (" << std::hex << properties + << "), interpreted as (" << (properties ^ _prop) << ")."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); } return propList; } @@ -173,7 +173,9 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, auto ctx = unwrap(CRef); if (!(dev && ctx)) { - /* \todo handle error */ + error_handler("Cannot create queue from DPCTLSyclContextRef and " + "DPCTLSyclDeviceRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return q; } auto propList = create_property_list(properties); @@ -183,20 +185,16 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, auto Queue = new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler), *propList); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else if (properties) { try { auto Queue = new queue(*ctx, *dev, *propList); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else if (handler) { @@ -204,20 +202,16 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, auto Queue = new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler)); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else { try { auto Queue = new queue(*ctx, *dev); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } @@ -234,7 +228,8 @@ DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef, auto Device = unwrap(DRef); if (!Device) { - std::cerr << "Cannot create queue from NULL device reference.\n"; + error_handler("Cannot create queue from NULL device reference.", + __FILE__, __func__, __LINE__); return QRef; } // Check if a cached default context exists for the device. @@ -248,8 +243,8 @@ DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef, try { ContextPtr = new context(*Device); CRef = wrap(ContextPtr); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); delete ContextPtr; return QRef; } @@ -280,14 +275,14 @@ DPCTLQueue_Copy(__dpctl_keep const DPCTLSyclQueueRef QRef) try { auto CopiedQueue = new queue(*Queue); return wrap(CopiedQueue); - } catch (std::bad_alloc &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Can not copy DPCTLSyclQueueRef as input is a nullptr" - << std::endl; + error_handler("Cannot copy DPCTLSyclQueueRef as input is a nullptr", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -295,9 +290,11 @@ DPCTLQueue_Copy(__dpctl_keep const DPCTLSyclQueueRef QRef) bool DPCTLQueue_AreEq(__dpctl_keep const DPCTLSyclQueueRef QRef1, __dpctl_keep const DPCTLSyclQueueRef QRef2) { - if (!(QRef1 && QRef2)) - // \todo handle error + if (!(QRef1 && QRef2)) { + error_handler("DPCTLSyclQueueRefs are nullptr.", __FILE__, __func__, + __LINE__); return false; + } return (*unwrap(QRef1) == *unwrap(QRef2)); } @@ -308,9 +305,8 @@ DPCTLSyclBackendType DPCTLQueue_GetBackend(__dpctl_keep DPCTLSyclQueueRef QRef) try { auto C = Q->get_context(); return DPCTLContext_GetBackend(wrap(&C)); - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; - // store error message + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return DPCTL_UNKNOWN_BACKEND; } } @@ -327,12 +323,13 @@ DPCTLQueue_GetDevice(__dpctl_keep const DPCTLSyclQueueRef QRef) try { auto Device = new device(Q->get_device()); DRef = wrap(Device); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else { - std::cerr << "Could not get the device for this queue.\n"; + error_handler("Could not get the device for this queue.", __FILE__, + __func__, __LINE__); } return DRef; } @@ -345,7 +342,8 @@ DPCTLQueue_GetContext(__dpctl_keep const DPCTLSyclQueueRef QRef) if (Q) CRef = wrap(new context(Q->get_context())); else { - std::cerr << "Could not get the context for this queue.\n"; + error_handler("Could not get the context for this queue.", __FILE__, + __func__, __LINE__); } return CRef; } @@ -374,7 +372,6 @@ DPCTLQueue_SubmitRange(__dpctl_keep const DPCTLSyclKernelRef KRef, for (auto i = 0ul; i < NArgs; ++i) { // \todo add support for Sycl buffers - // \todo handle errors properly if (!set_kernel_arg(cgh, i, Args[i], ArgTypes[i])) exit(1); } @@ -390,17 +387,12 @@ DPCTLQueue_SubmitRange(__dpctl_keep const DPCTLSyclKernelRef KRef, *Kernel); break; default: - // \todo handle the error throw std::runtime_error("Range cannot be greater than three " "dimensions."); } }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -432,7 +424,6 @@ DPCTLQueue_SubmitNDRange(__dpctl_keep const DPCTLSyclKernelRef KRef, for (auto i = 0ul; i < NArgs; ++i) { // \todo add support for Sycl buffers - // \todo handle errors properly if (!set_kernel_arg(cgh, i, Args[i], ArgTypes[i])) exit(1); } @@ -452,17 +443,12 @@ DPCTLQueue_SubmitNDRange(__dpctl_keep const DPCTLSyclKernelRef KRef, *Kernel); break; default: - // \todo handle the error throw std::runtime_error("Range cannot be greater than three " "dimensions."); } }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -479,8 +465,7 @@ void DPCTLQueue_Wait(__dpctl_keep DPCTLSyclQueueRef QRef) SyclQueue->wait(); } else { - // todo: log error - std::cerr << "Argument QRef is NULL" << '\n'; + error_handler("Argument QRef is NULL.", __FILE__, __func__, __LINE__); } } @@ -494,16 +479,15 @@ DPCTLSyclEventRef DPCTLQueue_Memcpy(__dpctl_keep const DPCTLSyclQueueRef QRef, sycl::event ev; try { ev = Q->memcpy(Dest, Src, Count); - } catch (const sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "QRef passed to memcpy was NULL" << '\n'; + error_handler("QRef passed to memcpy was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -518,23 +502,21 @@ DPCTLSyclEventRef DPCTLQueue_Prefetch(__dpctl_keep DPCTLSyclQueueRef QRef, sycl::event ev; try { ev = Q->prefetch(Ptr, Count); - } catch (sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "Attempt to prefetch USM-allocation at nullptr" - << '\n'; + error_handler("Attempt to prefetch USM-allocation at nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } } else { - // todo: log error - std::cerr << "QRef passed to prefetch was NULL" << '\n'; + error_handler("QRef passed to prefetch was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -549,16 +531,15 @@ DPCTLSyclEventRef DPCTLQueue_MemAdvise(__dpctl_keep DPCTLSyclQueueRef QRef, sycl::event ev; try { ev = Q->mem_advise(Ptr, Count, static_cast(Advice)); - } catch (const sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "QRef passed to prefetch was NULL" << '\n'; + error_handler("QRef passed to prefetch was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -591,8 +572,7 @@ size_t DPCTLQueue_Hash(__dpctl_keep const DPCTLSyclQueueRef QRef) return hash_fn(*Q); } else { - // todo: log error - std::cerr << "Argument QRef is null" << '\n'; + error_handler("Argument QRef is NULL.", __FILE__, __func__, __LINE__); return 0; } } @@ -614,20 +594,15 @@ __dpctl_give DPCTLSyclEventRef DPCTLQueue_SubmitBarrierForEvents( cgh.barrier(); }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(e)); } else { - // todo: log error - std::cerr << "Argument QRef is null" << '\n'; + error_handler("Argument QRef is NULL", __FILE__, __func__, __LINE__); return nullptr; } } From 4b1700e86bd47734f8dcfb266225f94eb00ce835 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 16:30:48 -0600 Subject: [PATCH 057/229] Use error_handler in dpctl_sycl_queue_manager.cpp --- .../source/dpctl_sycl_queue_manager.cpp | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp index c43f364cf9..4bd7a7d3ea 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp @@ -24,6 +24,7 @@ /// //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_manager.h" +#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_manager.h" #include /* SYCL headers */ @@ -56,16 +57,15 @@ struct QueueManager qs.emplace_back(*unwrap(CRef), *unwrap(DRef)); } else { - std::cerr << "Fatal Error: No cached context for default " - "device.\n"; + error_handler("Fatal Error: No cached context for default " + "device.", + __FILE__, __func__, __LINE__); std::terminate(); } delete unwrap(DRef); delete unwrap(CRef); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return qs; @@ -85,8 +85,8 @@ bool DPCTLQueueMgr_GlobalQueueIsCurrent() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "Error: No global queue found.\n"; + error_handler("Error: No global queue found.", __FILE__, __func__, + __LINE__); return false; } // The first entry of the QueueStack is always the global queue. If there @@ -104,8 +104,8 @@ DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "No currently active queues.\n"; + error_handler("No currently active queues.", __FILE__, __func__, + __LINE__); return nullptr; } auto last = qs.size() - 1; @@ -117,8 +117,8 @@ bool DPCTLQueueMgr_IsCurrentQueue(__dpctl_keep const DPCTLSyclQueueRef QRef) { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "No currently active queues.\n"; + error_handler("No currently active queues.", __FILE__, __func__, + __LINE__); return false; } auto last = qs.size() - 1; @@ -135,8 +135,9 @@ void DPCTLQueueMgr_SetGlobalQueue(__dpctl_keep const DPCTLSyclQueueRef qRef) qs[0] = *unwrap(qRef); } else { - // TODO: This should be an error and we should not fail silently. - std::cerr << "Error: Failed to set the global queue.\n"; + error_handler("Error: Failed to set the global queue.", __FILE__, + __func__, __LINE__); + std::terminate(); } } @@ -148,8 +149,9 @@ void DPCTLQueueMgr_PushQueue(__dpctl_keep const DPCTLSyclQueueRef qRef) qs.emplace_back(*unwrap(qRef)); } else { - // TODO: This should be an error and we should not fail silently. - std::cerr << "Error: Failed to set the current queue.\n"; + error_handler("Error: Failed to set the current queue.", __FILE__, + __func__, __LINE__); + std::terminate(); } } @@ -162,7 +164,7 @@ void DPCTLQueueMgr_PopQueue() // The first entry in the QueueStack is the global queue, and should not be // removed. if (qs.size() <= 1) { - std::cerr << "No queue to pop.\n"; + error_handler("No queue to pop.", __FILE__, __func__, __LINE__); return; } qs.pop_back(); @@ -172,8 +174,8 @@ size_t DPCTLQueueMgr_GetQueueStackSize() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "Error: No global queue found.\n"; + error_handler("Error: No global queue found.", __FILE__, __func__, + __LINE__); return -1; } // The first entry of the QueueStack is always the global queue. If there From 3e0c061916805e981aca39a6f12d6b2471bf59b4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 28 Nov 2021 23:55:27 -0600 Subject: [PATCH 058/229] Removed uses of deprecated exception --- .../helper/include/dpctl_string_utils.hpp | 6 +-- .../tests/test_sycl_device_subdevices.cpp | 54 +++++-------------- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/libsyclinterface/helper/include/dpctl_string_utils.hpp b/libsyclinterface/helper/include/dpctl_string_utils.hpp index aa1a8a9535..735f972128 100644 --- a/libsyclinterface/helper/include/dpctl_string_utils.hpp +++ b/libsyclinterface/helper/include/dpctl_string_utils.hpp @@ -21,6 +21,7 @@ /// \file /// Helper function to convert a C++ string to a C string. //===----------------------------------------------------------------------===// +#include "../helper/include/dpctl_error_handlers.h" #include #include #include @@ -55,9 +56,8 @@ cstring_from_string(const std::string &str) // to be null-terminated and the copy function is asked to // copy enough characters to include that null-character. cstr[cstr_len - 1] = '\0'; - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return cstr; diff --git a/libsyclinterface/tests/test_sycl_device_subdevices.cpp b/libsyclinterface/tests/test_sycl_device_subdevices.cpp index 1979826bfd..14252bdc88 100644 --- a/libsyclinterface/tests/test_sycl_device_subdevices.cpp +++ b/libsyclinterface/tests/test_sycl_device_subdevices.cpp @@ -174,13 +174,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityNuma) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -210,13 +205,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL4Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -246,13 +236,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL3Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -282,13 +267,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL2Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -318,13 +298,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL1Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -355,13 +330,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { From 3d9555748a7ec4d90f9297cccfd9fbb1a9aa3a5f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 10:03:33 -0600 Subject: [PATCH 059/229] Added tests for null KernelRef argument --- libsyclinterface/tests/test_sycl_kernel_interface.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_kernel_interface.cpp b/libsyclinterface/tests/test_sycl_kernel_interface.cpp index 3b984a4f11..3a5f5b5f0f 100644 --- a/libsyclinterface/tests/test_sycl_kernel_interface.cpp +++ b/libsyclinterface/tests/test_sycl_kernel_interface.cpp @@ -125,6 +125,14 @@ TEST_P(TestDPCTLSyclKernelInterface, CheckGetNumArgs) DPCTLKernel_Delete(AxpyKernel); } +TEST_P(TestDPCTLSyclKernelInterface, CheckNullPtrArg) +{ + DPCTLSyclKernelRef AddKernel = nullptr; + + ASSERT_EQ(DPCTLKernel_GetNumArgs(AddKernel), -1); + ASSERT_EQ(DPCTLKernel_GetFunctionName(AddKernel), nullptr); +} + INSTANTIATE_TEST_SUITE_P(TestKernelInterfaceFunctions, TestDPCTLSyclKernelInterface, ::testing::Values("opencl:gpu:0", "opencl:cpu:0")); From 95c85f43dc88cdba6cdc6ecfb5587e1f26ec23eb Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 11:41:34 -0600 Subject: [PATCH 060/229] More test with null pointers --- .../tests/test_sycl_program_interface.cpp | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_program_interface.cpp b/libsyclinterface/tests/test_sycl_program_interface.cpp index 3fbe154f28..a6970c2ce1 100644 --- a/libsyclinterface/tests/test_sycl_program_interface.cpp +++ b/libsyclinterface/tests/test_sycl_program_interface.cpp @@ -98,17 +98,37 @@ TEST_P(TestDPCTLSyclProgramInterface, ChkCreateFromSpirv) ASSERT_TRUE(PRef != nullptr); ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "add")); ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); + ASSERT_FALSE(DPCTLProgram_HasKernel(PRef, nullptr)); +} + +TEST_P(TestDPCTLSyclProgramInterface, ChkHasKernelNullProgram) +{ + + DPCTLSyclProgramRef NullRef = nullptr; + ASSERT_FALSE(DPCTLProgram_HasKernel(NullRef, "add")); } TEST_P(TestDPCTLSyclProgramInterface, ChkGetKernel) { auto AddKernel = DPCTLProgram_GetKernel(PRef, "add"); auto AxpyKernel = DPCTLProgram_GetKernel(PRef, "axpy"); + auto NullKernel = DPCTLProgram_GetKernel(PRef, nullptr); ASSERT_TRUE(AddKernel != nullptr); ASSERT_TRUE(AxpyKernel != nullptr); + ASSERT_TRUE(NullKernel == nullptr); DPCTLKernel_Delete(AddKernel); DPCTLKernel_Delete(AxpyKernel); + EXPECT_NO_FATAL_FAILURE(DPCTLKernel_Delete(NullKernel)); +} + +TEST_P(TestDPCTLSyclProgramInterface, ChkGetKernelNullProgram) +{ + DPCTLSyclProgramRef NullRef = nullptr; + DPCTLSyclKernelRef KRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(KRef = DPCTLProgram_GetKernel(NullRef, "add")); + EXPECT_TRUE(KRef == nullptr); } struct TestOCLProgramFromSource : public ::testing::Test @@ -163,6 +183,23 @@ TEST_F(TestOCLProgramFromSource, CheckCreateFromOCLSource) ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); } +TEST_F(TestOCLProgramFromSource, CheckCreateFromOCLSourceNull) +{ + const char *InvalidCLProgramStr = R"CLC( + kernel void invalid(global foo* a, global bar* b) { + size_t index = get_global_id(0); + b[index] = a[index]; + } + )CLC"; + DPCTLSyclProgramRef PRef = nullptr; + + if (!DRef) + GTEST_SKIP_("Skipping as no OpenCL GPU device found.\n"); + + EXPECT_NO_FATAL_FAILURE(PRef = DPCTLProgram_CreateFromOCLSource( + CRef, InvalidCLProgramStr, CompileOpts);); +} + TEST_F(TestOCLProgramFromSource, CheckGetKernelOCLSource) { if (!DRef) From 196d46ebf575349a7f3339e59b2caed13bee0c10 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 11:48:27 -0600 Subject: [PATCH 061/229] Null argument tests for sycl_platform --- .../tests/test_sycl_platform_interface.cpp | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_platform_interface.cpp b/libsyclinterface/tests/test_sycl_platform_interface.cpp index 95eaec4ee1..dc874b317e 100644 --- a/libsyclinterface/tests/test_sycl_platform_interface.cpp +++ b/libsyclinterface/tests/test_sycl_platform_interface.cpp @@ -115,6 +115,58 @@ struct TestDPCTLSyclPlatformInterface } }; +struct TestDPCTLSyclPlatformNull : public ::testing::Test +{ + DPCTLSyclPlatformRef NullPRef = nullptr; + DPCTLSyclDeviceSelectorRef NullDSRef = nullptr; + + TestDPCTLSyclPlatformNull() = default; + ~TestDPCTLSyclPlatformNull() = default; +}; + +TEST_F(TestDPCTLSyclPlatformNull, ChkCopy) +{ + DPCTLSyclPlatformRef Copied_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_PRef = DPCTLPlatform_Copy(NullPRef)); + ASSERT_TRUE(Copied_PRef == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkCreateFromSelector) +{ + DPCTLSyclPlatformRef Created_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Created_PRef = + DPCTLPlatform_CreateFromSelector(NullDSRef)); + ASSERT_TRUE(Created_PRef == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetBackend) +{ + DPCTLSyclBackendType BTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + EXPECT_NO_FATAL_FAILURE(BTy = DPCTLPlatform_GetBackend(NullPRef)); + ASSERT_TRUE(BTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetName) +{ + const char *name = nullptr; + EXPECT_NO_FATAL_FAILURE(name = DPCTLPlatform_GetName(NullPRef)); + ASSERT_TRUE(name == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetVendor) +{ + const char *vendor = nullptr; + EXPECT_NO_FATAL_FAILURE(vendor = DPCTLPlatform_GetVendor(NullPRef)); + ASSERT_TRUE(vendor == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetVersion) +{ + const char *version = nullptr; + EXPECT_NO_FATAL_FAILURE(version = DPCTLPlatform_GetVersion(NullPRef)); + ASSERT_TRUE(version == nullptr); +} + struct TestDPCTLSyclDefaultPlatform : public ::testing::Test { DPCTLSyclPlatformRef PRef = nullptr; @@ -163,11 +215,26 @@ TEST_P(TestDPCTLSyclPlatformInterface, ChkCopy) EXPECT_NO_FATAL_FAILURE(DPCTLPlatform_Delete(Copied_PRef)); } +TEST_P(TestDPCTLSyclPlatformInterface, ChkCopyNullArg) +{ + DPCTLSyclPlatformRef Null_PRef = nullptr; + DPCTLSyclPlatformRef Copied_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_PRef = DPCTLPlatform_Copy(Null_PRef)); + EXPECT_FALSE(bool(Copied_PRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLPlatform_Delete(Copied_PRef)); +} + TEST_P(TestDPCTLSyclPlatformInterface, ChkPrintInfo) { EXPECT_NO_FATAL_FAILURE(DPCTLPlatformMgr_PrintInfo(PRef, 0)); } +TEST_P(TestDPCTLSyclPlatformInterface, ChkPrintInfoNullArg) +{ + DPCTLSyclPlatformRef Null_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DPCTLPlatformMgr_PrintInfo(Null_PRef, 0)); +} + TEST_F(TestDPCTLSyclDefaultPlatform, ChkGetName) { check_platform_name(PRef); From 98f996ff170dd9f6d2f6f3cfc29bc57e4ad56c6b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 12:26:33 -0600 Subject: [PATCH 062/229] tests of queue_manager for null arguments --- .../tests/test_sycl_queue_manager.cpp | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_queue_manager.cpp b/libsyclinterface/tests/test_sycl_queue_manager.cpp index d655d5090c..433dc89b19 100644 --- a/libsyclinterface/tests/test_sycl_queue_manager.cpp +++ b/libsyclinterface/tests/test_sycl_queue_manager.cpp @@ -314,3 +314,42 @@ TEST_F(TestDPCTLQueueMgrFeatures, GTEST_SKIP_("OpenCL CPU devices are needed, but were not found."); } } + +struct TestDPCTLQueueMgrNullArgs : public ::testing::Test +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + + TestDPCTLQueueMgrNullArgs() {} + ~TestDPCTLQueueMgrNullArgs() {} +}; + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkGlobalQueueIsCurrent) +{ + bool res = true; + EXPECT_NO_FATAL_FAILURE(res = DPCTLQueueMgr_GlobalQueueIsCurrent()); + ASSERT_TRUE(res == true || res == false); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkIsCurrentQueue) +{ + bool res = true; + EXPECT_NO_FATAL_FAILURE(res = DPCTLQueueMgr_IsCurrentQueue(Null_QRef)); + ASSERT_FALSE(res); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkSetGlobalQueue) +{ + EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkPushGlobalQueue) +{ + EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkGetQueueStackSize) +{ + size_t n = 0; + EXPECT_NO_FATAL_FAILURE(n = DPCTLQueueMgr_GetQueueStackSize()); + ASSERT_TRUE(n < size_t(-1)); +} From c0d4f8805fa82322db735cb7258317d7b30b5286 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 13:33:37 -0600 Subject: [PATCH 063/229] DPCTLQueueMgr_IsCurrentQueue must check that QRef is not null before dereferencing --- libsyclinterface/source/dpctl_sycl_queue_manager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp index 4bd7a7d3ea..5cd924bdd4 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp @@ -115,6 +115,9 @@ DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() // Relies on sycl::queue class' operator= to check for equivalent of queues. bool DPCTLQueueMgr_IsCurrentQueue(__dpctl_keep const DPCTLSyclQueueRef QRef) { + if (!QRef) { + return false; + } auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { error_handler("No currently active queues.", __FILE__, __func__, From c7972a3bcb5c4b598a01360caf8a6fd2416501c2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 13:47:02 -0600 Subject: [PATCH 064/229] exclude tests which lead to call to std::terminate --- libsyclinterface/tests/test_sycl_queue_manager.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_queue_manager.cpp b/libsyclinterface/tests/test_sycl_queue_manager.cpp index 433dc89b19..69c339e0a8 100644 --- a/libsyclinterface/tests/test_sycl_queue_manager.cpp +++ b/libsyclinterface/tests/test_sycl_queue_manager.cpp @@ -337,6 +337,7 @@ TEST_F(TestDPCTLQueueMgrNullArgs, ChkIsCurrentQueue) ASSERT_FALSE(res); } +#if 0 TEST_F(TestDPCTLQueueMgrNullArgs, ChkSetGlobalQueue) { EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); @@ -346,6 +347,7 @@ TEST_F(TestDPCTLQueueMgrNullArgs, ChkPushGlobalQueue) { EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); } +#endif TEST_F(TestDPCTLQueueMgrNullArgs, ChkGetQueueStackSize) { From 4a3d9ef7f02c63dffa86eb05c0c340fed748ed44 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 13:56:19 -0600 Subject: [PATCH 065/229] DPCTLProgram_HasKernel should handle nullptr in KernelName --- libsyclinterface/source/dpctl_sycl_program_interface.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index 7e24d3fd34..b01e82e8e1 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -317,6 +317,11 @@ bool DPCTLProgram_HasKernel(__dpctl_keep DPCTLSyclProgramRef PRef, error_handler("Input PRef is nullptr", __FILE__, __func__, __LINE__); return false; } + if (!KernelName) { + error_handler("Input KernelName is nullptr", __FILE__, __func__, + __LINE__); + return false; + } auto SyclProgram = unwrap(PRef); try { From b2859cf20a1123e5ab8c21137a10b0b891ac82ff Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 14:26:36 -0600 Subject: [PATCH 066/229] Tests for null args for DPCTLContext_* Added checks for null args in DPCTLContext_Delete, and in DPCTLContext_GetBackend --- .../source/dpctl_sycl_context_interface.cpp | 4 + .../tests/test_sycl_context_interface.cpp | 79 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/libsyclinterface/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp index d6a74833dd..62d488d935 100644 --- a/libsyclinterface/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -183,6 +183,10 @@ void DPCTLContext_Delete(__dpctl_take DPCTLSyclContextRef CtxRef) DPCTLSyclBackendType DPCTLContext_GetBackend(__dpctl_keep const DPCTLSyclContextRef CtxRef) { + if (!CtxRef) { + return DPCTL_UNKNOWN_BACKEND; + } + auto BE = unwrap(CtxRef)->get_platform().get_backend(); switch (BE) { diff --git a/libsyclinterface/tests/test_sycl_context_interface.cpp b/libsyclinterface/tests/test_sycl_context_interface.cpp index fb42725f23..1d609671a0 100644 --- a/libsyclinterface/tests/test_sycl_context_interface.cpp +++ b/libsyclinterface/tests/test_sycl_context_interface.cpp @@ -211,3 +211,82 @@ INSTANTIATE_TEST_SUITE_P(DPCTLContextTests, "gpu:0", "gpu:1", "1")); + +struct TestDPCTLContextNullArgs : public ::testing::Test +{ + DPCTLSyclContextRef Null_CRef = nullptr; + DPCTLSyclDeviceRef Null_DRef = nullptr; + DPCTLDeviceVectorRef Null_DVRef = nullptr; + TestDPCTLContextNullArgs() = default; + ~TestDPCTLContextNullArgs() = default; +}; + +TEST_F(TestDPCTLContextNullArgs, ChkCreate) +{ + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(CRef = DPCTLContext_Create(Null_DRef, nullptr, 0)); + ASSERT_FALSE(bool(CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkCreateFromDevices) +{ + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + CRef = DPCTLContext_CreateFromDevices(Null_DVRef, nullptr, 0)); + ASSERT_FALSE(bool(CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkAreEq) +{ + DPCTLSyclContextRef Null_C2Ref = nullptr; + bool are_eq = true; + EXPECT_NO_FATAL_FAILURE(are_eq = DPCTLContext_AreEq(Null_CRef, Null_C2Ref)); + ASSERT_FALSE(are_eq); +} + +TEST_F(TestDPCTLContextNullArgs, ChkCopy) +{ + DPCTLSyclContextRef Copied_CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_CRef = DPCTLContext_Copy(Null_CRef)); + ASSERT_FALSE(bool(Copied_CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkGetDevices) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DVRef = DPCTLContext_GetDevices(Null_CRef)); + ASSERT_FALSE(bool(DVRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkDeviceCount) +{ + size_t count = -1; + EXPECT_NO_FATAL_FAILURE(count = DPCTLContext_DeviceCount(Null_CRef)); + ASSERT_TRUE(count == 0); +} + +TEST_F(TestDPCTLContextNullArgs, ChkIsHost) +{ + bool is_host = true; + EXPECT_NO_FATAL_FAILURE(is_host = DPCTLContext_IsHost(Null_CRef)); + ASSERT_FALSE(is_host); +} + +TEST_F(TestDPCTLContextNullArgs, ChkHash) +{ + size_t hash = 0; + EXPECT_NO_FATAL_FAILURE(hash = DPCTLContext_Hash(Null_CRef)); + ASSERT_TRUE(hash == 0); +} + +TEST_F(TestDPCTLContextNullArgs, ChkGetBackend) +{ + DPCTLSyclBackendType BTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + EXPECT_NO_FATAL_FAILURE(BTy = DPCTLContext_GetBackend(Null_CRef)); + ASSERT_TRUE(BTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestDPCTLContextNullArgs, ChkDelete) +{ + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(Null_CRef)); +} From 76a88c14053427963b1239b8dec59339b3e43554 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 16:11:49 -0600 Subject: [PATCH 067/229] Added group of tests for null args to DPCTLDevice_* --- .../tests/test_sycl_device_interface.cpp | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_device_interface.cpp b/libsyclinterface/tests/test_sycl_device_interface.cpp index 25ff7c3f47..b33e2c566f 100644 --- a/libsyclinterface/tests/test_sycl_device_interface.cpp +++ b/libsyclinterface/tests/test_sycl_device_interface.cpp @@ -390,3 +390,286 @@ INSTANTIATE_TEST_SUITE_P(DPCTLDeviceFns, "gpu:0", "gpu:1", "1")); + +struct TestDPCTLSyclDeviceNullArgs : public ::testing::Test +{ + DPCTLSyclDeviceRef Null_DRef = nullptr; + DPCTLSyclDeviceSelectorRef Null_DSRef = nullptr; +}; + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCopy) +{ + DPCTLSyclDeviceRef Copied_DRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_DRef = DPCTLDevice_Copy(Null_DRef)); + ASSERT_FALSE(bool(Copied_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateFromSelector) +{ + DPCTLSyclDeviceRef Created_DRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Created_DRef = + DPCTLDevice_CreateFromSelector(Null_DSRef)); + ASSERT_FALSE(bool(Created_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkDelete) +{ + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(Null_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetDeviceType) +{ + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE; + EXPECT_NO_FATAL_FAILURE(DTy = DPCTLDevice_GetDeviceType(Null_DRef)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsAccelerator) +{ + bool is_acc = true; + EXPECT_NO_FATAL_FAILURE(is_acc = DPCTLDevice_IsAccelerator(Null_DRef)); + ASSERT_FALSE(is_acc); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsCPU) +{ + bool is_cpu = true; + EXPECT_NO_FATAL_FAILURE(is_cpu = DPCTLDevice_IsCPU(Null_DRef)); + ASSERT_FALSE(is_cpu); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsGPU) +{ + bool is_gpu = true; + EXPECT_NO_FATAL_FAILURE(is_gpu = DPCTLDevice_IsGPU(Null_DRef)); + ASSERT_FALSE(is_gpu); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsHost) +{ + bool is_host = true; + EXPECT_NO_FATAL_FAILURE(is_host = DPCTLDevice_IsHost(Null_DRef)); + ASSERT_FALSE(is_host); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxComputeUnits) +{ + uint32_t mcu = -1; + EXPECT_NO_FATAL_FAILURE(mcu = DPCTLDevice_GetMaxComputeUnits(Null_DRef)); + ASSERT_TRUE(mcu == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetGlobalMemSize) +{ + uint64_t gmsz = -1; + EXPECT_NO_FATAL_FAILURE(gmsz = DPCTLDevice_GetGlobalMemSize(Null_DRef)); + ASSERT_TRUE(gmsz == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetLocalMemSize) +{ + uint64_t lmsz = -1; + EXPECT_NO_FATAL_FAILURE(lmsz = DPCTLDevice_GetLocalMemSize(Null_DRef)); + ASSERT_TRUE(lmsz == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkItemDims) +{ + uint32_t md = -1; + EXPECT_NO_FATAL_FAILURE(md = DPCTLDevice_GetMaxWorkItemDims(Null_DRef)); + ASSERT_TRUE(md == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkItemSizes) +{ + size_t *sz = nullptr; + EXPECT_NO_FATAL_FAILURE(sz = DPCTLDevice_GetMaxWorkItemSizes(Null_DRef)); + ASSERT_TRUE(sz == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkGroupSize) +{ + size_t m = -1; + EXPECT_NO_FATAL_FAILURE(m = DPCTLDevice_GetMaxWorkGroupSize(Null_DRef)); + ASSERT_TRUE(m == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxNumSubGroups) +{ + uint32_t nsg = -1; + EXPECT_NO_FATAL_FAILURE(nsg = DPCTLDevice_GetMaxNumSubGroups(Null_DRef)); + ASSERT_TRUE(nsg == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetPlatform) +{ + DPCTLSyclPlatformRef PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(PRef = DPCTLDevice_GetPlatform(Null_DRef)); + ASSERT_TRUE(PRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkName) +{ + const char *name = nullptr; + EXPECT_NO_FATAL_FAILURE(name = DPCTLDevice_GetName(Null_DRef)); + ASSERT_TRUE(name == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkVendor) +{ + const char *vendor = nullptr; + EXPECT_NO_FATAL_FAILURE(vendor = DPCTLDevice_GetVendor(Null_DRef)); + ASSERT_TRUE(vendor == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkDriverVersion) +{ + const char *driver_version = nullptr; + EXPECT_NO_FATAL_FAILURE(driver_version = + DPCTLDevice_GetDriverVersion(Null_DRef)); + ASSERT_TRUE(driver_version == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsHostUnifiedMemory) +{ + bool is_hum = true; + EXPECT_NO_FATAL_FAILURE(is_hum = + DPCTLDevice_IsHostUnifiedMemory(Null_DRef)); + ASSERT_FALSE(is_hum); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkAreEq) +{ + bool are_eq = true; + EXPECT_NO_FATAL_FAILURE(are_eq = DPCTLDevice_AreEq(Null_DRef, Null_DRef)); + ASSERT_FALSE(are_eq); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkHasAspect) +{ + bool has_fp64 = true; + EXPECT_NO_FATAL_FAILURE(has_fp64 = DPCTLDevice_HasAspect( + Null_DRef, DPCTL_SyclAspectToDPCTLAspectType( + DPCTL_StrToAspectType("fp64")))); + ASSERT_FALSE(has_fp64); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxReadWriteImageArgs) +{ + uint32_t res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetMaxReadImageArgs(Null_DRef)); + ASSERT_TRUE(res == 0); + uint32_t wes = 0; + EXPECT_NO_FATAL_FAILURE(wes = DPCTLDevice_GetMaxWriteImageArgs(Null_DRef)); + ASSERT_TRUE(wes == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxImageDims) +{ + size_t res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage2dMaxWidth(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage2dMaxHeight(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxHeight(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxWidth(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxDepth(Null_DRef)); + ASSERT_TRUE(res == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetSubGroupIndependentForwardProgress) +{ + bool indep_pr = true; + EXPECT_NO_FATAL_FAILURE( + indep_pr = + DPCTLDevice_GetSubGroupIndependentForwardProgress(Null_DRef)); + ASSERT_FALSE(indep_pr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetPreferredVectorWidth) +{ + uint32_t w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthChar(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthShort(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthInt(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthLong(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthHalf(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthFloat(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthDouble(Null_DRef)); + ASSERT_TRUE(w == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetParentDevice) +{ + DPCTLSyclDeviceRef pDRef = nullptr; + EXPECT_NO_FATAL_FAILURE(pDRef = DPCTLDevice_GetParentDevice(Null_DRef)); + ASSERT_TRUE(pDRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesEqually) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesEqually(Null_DRef, 2)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesByCounts) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + size_t counts[2] = {1, 1}; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByCounts(Null_DRef, counts, 2)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesByAffinity) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity( + Null_DRef, DPCTLPartitionAffinityDomainType::not_applicable)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkHash) +{ + size_t hash = 0; + EXPECT_NO_FATAL_FAILURE(hash = DPCTLDevice_Hash(Null_DRef)); + ASSERT_TRUE(hash == 0); +} From 176702d8cfeb52a135881ed3cd3b7edddf2fb5b4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 16:24:55 -0600 Subject: [PATCH 068/229] Test DPCTLDeviceSelector_Score with null arguments --- .../tests/test_sycl_device_selector_interface.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_device_selector_interface.cpp b/libsyclinterface/tests/test_sycl_device_selector_interface.cpp index 71924d1023..592575b53a 100644 --- a/libsyclinterface/tests/test_sycl_device_selector_interface.cpp +++ b/libsyclinterface/tests/test_sycl_device_selector_interface.cpp @@ -223,6 +223,13 @@ TEST_F(TestDeviceSelectorInterface, ChkDPCTLGPUSelectorScore) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef_GPU)); EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef_CPU)); EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + DPCTLSyclDeviceSelectorRef Null_DSRef = nullptr; + DPCTLSyclDeviceRef Null_DRef = nullptr; + int score = 1; + EXPECT_NO_FATAL_FAILURE( + score = DPCTLDeviceSelector_Score(Null_DSRef, Null_DRef)); + ASSERT_TRUE(score < 0); } INSTANTIATE_TEST_SUITE_P(FilterSelectorCreation, From 9719434f884f7684ace5f5352a0cd768f3db6f43 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 16:30:36 -0600 Subject: [PATCH 069/229] test CreateFromSpirv with null context --- libsyclinterface/tests/test_sycl_program_interface.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_program_interface.cpp b/libsyclinterface/tests/test_sycl_program_interface.cpp index a6970c2ce1..6368a51801 100644 --- a/libsyclinterface/tests/test_sycl_program_interface.cpp +++ b/libsyclinterface/tests/test_sycl_program_interface.cpp @@ -101,6 +101,16 @@ TEST_P(TestDPCTLSyclProgramInterface, ChkCreateFromSpirv) ASSERT_FALSE(DPCTLProgram_HasKernel(PRef, nullptr)); } +TEST_P(TestDPCTLSyclProgramInterface, ChkCreateFromSpirvNull) +{ + DPCTLSyclContextRef Null_CRef = nullptr; + const void *null_spirv = nullptr; + DPCTLSyclProgramRef PRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + PRef = DPCTLProgram_CreateFromSpirv(Null_CRef, null_spirv, 0, nullptr)); + ASSERT_TRUE(PRef == nullptr); +} + TEST_P(TestDPCTLSyclProgramInterface, ChkHasKernelNullProgram) { From 5ba1e618977c20807b67ec0950fa9840684c24aa Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 16:37:42 -0600 Subject: [PATCH 070/229] Added test for SubmitNDRange --- .../tests/test_sycl_queue_submit.cpp | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_queue_submit.cpp b/libsyclinterface/tests/test_sycl_queue_submit.cpp index 04441347f9..1176e4014d 100644 --- a/libsyclinterface/tests/test_sycl_queue_submit.cpp +++ b/libsyclinterface/tests/test_sycl_queue_submit.cpp @@ -43,6 +43,7 @@ namespace { constexpr size_t SIZE = 1024; +static_assert(SIZE % 8 == 0); DEFINE_SIMPLE_CONVERSION_FUNCTIONS(void, DPCTLSyclUSMRef); } /* end of anonymous namespace */ @@ -127,6 +128,71 @@ TEST_F(TestQueueSubmit, CheckSubmitRange_saxpy) DPCTLDeviceSelector_Delete(DSRef); } +TEST_F(TestQueueSubmit, CheckSubmitNDRange_saxpy) +{ + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + DPCTLDeviceMgr_PrintDeviceInfo(DRef); + ASSERT_TRUE(DRef); + auto QRef = + DPCTLQueue_CreateForDevice(DRef, nullptr, DPCTL_DEFAULT_PROPERTY); + ASSERT_TRUE(QRef); + auto CRef = DPCTLQueue_GetContext(QRef); + ASSERT_TRUE(CRef); + auto PRef = DPCTLProgram_CreateFromSpirv(CRef, spirvBuffer.data(), + spirvFileSize, nullptr); + ASSERT_TRUE(PRef != nullptr); + ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); + auto AxpyKernel = DPCTLProgram_GetKernel(PRef, "axpy"); + + // Create the input args + auto a = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(a != nullptr); + auto b = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(b != nullptr); + auto c = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(c != nullptr); + + auto a_ptr = reinterpret_cast(unwrap(a)); + auto b_ptr = reinterpret_cast(unwrap(b)); + // Initialize a,b + for (auto i = 0ul; i < SIZE; ++i) { + a_ptr[i] = i + 1.0; + b_ptr[i] = i + 2.0; + } + + // Create kernel args for axpy + float d = 10.0; + size_t gRange[] = {1, 1, SIZE}; + size_t lRange[] = {1, 1, 8}; + void *args2[4] = {unwrap(a), unwrap(b), unwrap(c), (void *)&d}; + DPCTLKernelArgType addKernelArgTypes[] = {DPCTL_VOID_PTR, DPCTL_VOID_PTR, + DPCTL_VOID_PTR, DPCTL_FLOAT}; + DPCTLSyclEventRef events[1]; + events[0] = DPCTLEvent_Create(); + + auto ERef = + DPCTLQueue_SubmitNDRange(AxpyKernel, QRef, args2, addKernelArgTypes, 4, + gRange, lRange, 3, events, 1); + ASSERT_TRUE(ERef != nullptr); + DPCTLQueue_Wait(QRef); + + // clean ups + DPCTLEvent_Delete(ERef); + DPCTLKernel_Delete(AxpyKernel); + DPCTLfree_with_queue((DPCTLSyclUSMRef)a, QRef); + DPCTLfree_with_queue((DPCTLSyclUSMRef)b, QRef); + DPCTLfree_with_queue((DPCTLSyclUSMRef)c, QRef); + DPCTLQueue_Delete(QRef); + DPCTLContext_Delete(CRef); + DPCTLProgram_Delete(PRef); + DPCTLDevice_Delete(DRef); + DPCTLDeviceSelector_Delete(DSRef); +} + #ifndef DPCTL_COVERAGE namespace { From a7682b2d08a9cd879c7c68afe00bd8eb4c4e36ec Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 30 Nov 2021 18:57:31 -0600 Subject: [PATCH 071/229] Added null argument tests for DPCTL functions to work with USM allocations --- .../tests/test_sycl_usm_interface.cpp | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/libsyclinterface/tests/test_sycl_usm_interface.cpp b/libsyclinterface/tests/test_sycl_usm_interface.cpp index 414c37a565..c7c137c143 100644 --- a/libsyclinterface/tests/test_sycl_usm_interface.cpp +++ b/libsyclinterface/tests/test_sycl_usm_interface.cpp @@ -27,11 +27,13 @@ #include "Support/CBindingWrapping.h" #include "dpctl_sycl_context_interface.h" #include "dpctl_sycl_device_interface.h" +#include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_event_interface.h" #include "dpctl_sycl_queue_interface.h" #include "dpctl_sycl_queue_manager.h" #include "dpctl_sycl_usm_interface.h" #include +#include #include using namespace cl::sycl; @@ -163,3 +165,107 @@ TEST_F(TestDPCTLSyclUSMInterface, AlignedAllocHost) common_test_body(nbytes, Ptr, Q, "host"); DPCTLfree_with_queue(Ptr, Q); } + +struct TestDPCTLSyclUSMNullArgs : public ::testing::Test +{ +}; + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkMalloc) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + void *ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_shared(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_device(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_host(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkAlignedAlloc) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + void *ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(ptr = + DPCTLaligned_alloc_shared(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = + DPCTLaligned_alloc_device(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLaligned_alloc_host(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkFree) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + DPCTLSyclUSMRef ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_queue(ptr, Null_QRef)); + + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLSyclQueueRef QRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + EXPECT_NO_FATAL_FAILURE(QRef = DPCTLQueue_CreateForDevice( + DRef, nullptr, DPCTL_DEFAULT_PROPERTY)); + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_queue(ptr, QRef)); + + DPCTLSyclContextRef Null_CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_context(ptr, Null_CRef)); + + DPCTLSyclContextRef CRef = DPCTLQueue_GetContext(QRef); + EXPECT_NO_FATAL_FAILURE(DPCTLQueue_Delete(QRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_context(ptr, CRef)); + + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(CRef)); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkPointerQueries) +{ + DPCTLSyclContextRef Null_CRef = nullptr; + DPCTLSyclUSMRef Null_MRef = nullptr; + const char *t = nullptr; + auto is_unknown = [=](const char *t) -> bool { + return strncmp(t, "unknown", 7) == 0; + }; + + EXPECT_NO_FATAL_FAILURE(t = DPCTLUSM_GetPointerType(Null_MRef, Null_CRef)); + ASSERT_TRUE(is_unknown(t)); + + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + ASSERT_TRUE(bool(DRef)); + + EXPECT_NO_FATAL_FAILURE(CRef = DPCTLContext_Create(DRef, nullptr, 0)); + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + t = nullptr; + EXPECT_NO_FATAL_FAILURE(t = DPCTLUSM_GetPointerType(Null_MRef, CRef)); + ASSERT_TRUE(is_unknown(t)); + + DPCTLSyclDeviceRef D2Ref = nullptr; + EXPECT_NO_FATAL_FAILURE(D2Ref = DPCTLUSM_GetPointerDevice(Null_MRef, CRef)); + ASSERT_TRUE(D2Ref == nullptr); + + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(CRef)); + + EXPECT_NO_FATAL_FAILURE( + D2Ref = DPCTLUSM_GetPointerDevice(Null_MRef, Null_CRef)); + ASSERT_TRUE(D2Ref == nullptr); +} From 05a57e34c2ab4f4c7a35c9f68969727ed877abeb Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 09:22:13 -0600 Subject: [PATCH 072/229] Add a test file to cover functions in dpctl_utils_helper --- libsyclinterface/tests/test_helper.cpp | 201 +++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 libsyclinterface/tests/test_helper.cpp diff --git a/libsyclinterface/tests/test_helper.cpp b/libsyclinterface/tests/test_helper.cpp new file mode 100644 index 0000000000..76c8d50932 --- /dev/null +++ b/libsyclinterface/tests/test_helper.cpp @@ -0,0 +1,201 @@ +//===--- test_helper.cpp - Test cases for helper functions ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file has unit test cases for functions defined in +/// helper/include/dpctl_utils_helper.h. +/// +//===----------------------------------------------------------------------===// + +#include "../helper/include/dpctl_utils_helper.h" +#include "Config/dpctl_config.h" +#include +#include +#include + +struct TestHelperFns : public ::testing::Test +{ +}; + +TEST_F(TestHelperFns, ChkDeviceTypeToStr) +{ + std::string res; + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::cpu)); + ASSERT_TRUE(res == "cpu"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::gpu)); + ASSERT_TRUE(res == "gpu"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::host)); + ASSERT_TRUE(res == "host"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::custom)); + ASSERT_TRUE(res == "custom"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::accelerator)); + ASSERT_TRUE(res == "accelerator"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::all)); + ASSERT_TRUE(res == "unknown"); +} + +TEST_F(TestHelperFns, ChkStrToDeviceType) +{ + sycl::info::device_type dev_type = sycl::info::device_type::automatic; + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("cpu")); + ASSERT_TRUE(dev_type == sycl::info::device_type::cpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("gpu")); + ASSERT_TRUE(dev_type == sycl::info::device_type::gpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("host")); + ASSERT_TRUE(dev_type == sycl::info::device_type::host); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("accelerator")); + ASSERT_TRUE(dev_type == sycl::info::device_type::accelerator); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("custom")); + ASSERT_TRUE(dev_type == sycl::info::device_type::custom); + + EXPECT_THROW(DPCTL_StrToDeviceType("invalid"), std::runtime_error); +} + +TEST_F(TestHelperFns, ChkDPCTLBackendTypeToSyclBackend) +{ + sycl::backend res = sycl::backend::level_zero; + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_CUDA)); + ASSERT_TRUE(res == sycl::backend::cuda); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_HOST)); + ASSERT_TRUE(res == sycl::backend::host); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_OPENCL)); + ASSERT_TRUE(res == sycl::backend::opencl); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_LEVEL_ZERO)); + ASSERT_TRUE(res == sycl::backend::level_zero); + + EXPECT_THROW(DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND), + std::runtime_error); +} + +TEST_F(TestHelperFns, ChkSyclBackendToDPCTLBackendType) +{ + DPCTLSyclBackendType DTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::level_zero)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_LEVEL_ZERO); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::opencl)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_OPENCL); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::host)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_HOST); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::cuda)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_CUDA); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::all)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestHelperFns, ChkDPCTLDeviceTypeToSyclDeviceType) +{ + sycl::info::device_type dev_type = sycl::info::device_type::automatic; + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_CPU)); + ASSERT_TRUE(dev_type == sycl::info::device_type::cpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_GPU)); + ASSERT_TRUE(dev_type == sycl::info::device_type::gpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_ACCELERATOR)); + ASSERT_TRUE(dev_type == sycl::info::device_type::accelerator); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_CUSTOM)); + ASSERT_TRUE(dev_type == sycl::info::device_type::custom); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_HOST_DEVICE)); + ASSERT_TRUE(dev_type == sycl::info::device_type::host); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_AUTOMATIC)); + ASSERT_TRUE(dev_type == sycl::info::device_type::automatic); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_ALL)); + ASSERT_TRUE(dev_type == sycl::info::device_type::all); +} + +TEST_F(TestHelperFns, SyclDeviceTypeToDPCTLDeviceType) +{ + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE; + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::cpu)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_CPU); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::gpu)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_GPU); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::host)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_HOST_DEVICE); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::accelerator)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_ACCELERATOR); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::automatic)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_AUTOMATIC); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::all)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_ALL); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::custom)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_CUSTOM); +} From fb73ef68c37164b265e9bc54226e408e18633cd7 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 10:02:28 -0600 Subject: [PATCH 073/229] Resolve the unused variable compiler warning --- libsyclinterface/tests/test_sycl_program_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libsyclinterface/tests/test_sycl_program_interface.cpp b/libsyclinterface/tests/test_sycl_program_interface.cpp index 6368a51801..13c4468a30 100644 --- a/libsyclinterface/tests/test_sycl_program_interface.cpp +++ b/libsyclinterface/tests/test_sycl_program_interface.cpp @@ -208,6 +208,7 @@ TEST_F(TestOCLProgramFromSource, CheckCreateFromOCLSourceNull) EXPECT_NO_FATAL_FAILURE(PRef = DPCTLProgram_CreateFromOCLSource( CRef, InvalidCLProgramStr, CompileOpts);); + ASSERT_TRUE(PRef == nullptr); } TEST_F(TestOCLProgramFromSource, CheckGetKernelOCLSource) From 2870d5e18127b9075916830992957a304f488a43 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 1 Dec 2021 10:02:58 -0600 Subject: [PATCH 074/229] Replace use of deprecated sycl::runtime_error with std::runtime_error in helper functions --- .../helper/source/dpctl_utils_helper.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp index 6cb0d977f5..212a5634fa 100644 --- a/libsyclinterface/helper/source/dpctl_utils_helper.cpp +++ b/libsyclinterface/helper/source/dpctl_utils_helper.cpp @@ -97,7 +97,7 @@ backend DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType BeTy) case DPCTLSyclBackendType::DPCTL_OPENCL: return backend::opencl; default: - throw runtime_error("Unsupported backend type", -1); + throw std::runtime_error("Unsupported backend type"); } } @@ -135,7 +135,7 @@ info::device_type DPCTL_DPCTLDeviceTypeToSyclDeviceType(DPCTLSyclDeviceType DTy) case DPCTLSyclDeviceType::DPCTL_HOST_DEVICE: return info::device_type::host; default: - throw runtime_error("Unsupported device type", -1); + throw std::runtime_error("Unsupported device type"); } } @@ -223,7 +223,7 @@ std::string DPCTL_AspectToStr(aspect aspectTy) ss << "usm_system_allocator"; break; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } return ss.str(); } @@ -290,7 +290,7 @@ aspect DPCTL_StrToAspectType(const std::string &aspectTyStr) } else { // \todo handle the error - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } return aspectTy; } @@ -335,7 +335,7 @@ aspect DPCTL_DPCTLAspectTypeToSyclAspect(DPCTLSyclAspectType AspectTy) case DPCTLSyclAspectType::usm_system_allocator: return aspect::usm_system_allocator; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } } @@ -379,7 +379,7 @@ DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(aspect Aspect) case aspect::usm_system_allocator: return DPCTLSyclAspectType::usm_system_allocator; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } } @@ -402,7 +402,7 @@ info::partition_affinity_domain DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( case DPCTLPartitionAffinityDomainType::next_partitionable: return info::partition_affinity_domain::next_partitionable; default: - throw runtime_error("Unsupported partition_affinity_domain type", -1); + throw std::runtime_error("Unsupported partition_affinity_domain type"); } } @@ -425,7 +425,7 @@ DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( case info::partition_affinity_domain::next_partitionable: return DPCTLPartitionAffinityDomainType::next_partitionable; default: - throw runtime_error("Unsupported partition_affinity_domain type", -1); + throw std::runtime_error("Unsupported partition_affinity_domain type"); } } From 7b191c91f9906ee0ff763d8c55c0880445a0bb20 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 22 Nov 2021 16:08:29 -0600 Subject: [PATCH 075/229] Family of make_Sycl*( Ref ) functions This is needed in order to be able to write pybind11 casters directly mapping sycl classes to `dpctl.Sycl*` classes for SyclContext, SyclQueue, SyclDevice and SyclEvent classes. --- dpctl/_sycl_context.pyx | 9 +++++++++ dpctl/_sycl_device.pyx | 9 +++++++++ dpctl/_sycl_event.pyx | 9 +++++++++ dpctl/_sycl_queue.pyx | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/dpctl/_sycl_context.pyx b/dpctl/_sycl_context.pyx index 404f27c99b..0499a21c06 100644 --- a/dpctl/_sycl_context.pyx +++ b/dpctl/_sycl_context.pyx @@ -485,3 +485,12 @@ cdef api DPCTLSyclContextRef get_context_ref(SyclContext ctx): :class:`dpctl.SyclContext` instance. """ return ctx.get_context_ref() + + +cdef api SyclContext make_SyclContext(DPCTLSyclContextRef CRef): + """ + C-API function to create :class:`dpctl.SyclContext` instance + from the given opaque context reference. + """ + cdef DPCTLSyclContextRef copied_CRef = DPCTLContext_Copy(CRef) + return SyclContext._create(copied_CRef) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 70302e9937..c32575b1ae 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -1136,3 +1136,12 @@ cdef api DPCTLSyclDeviceRef get_device_ref(SyclDevice dev): :class:`dpctl.SyclDevice` instance. """ return dev.get_device_ref() + + +cdef api SyclDevice make_SyclDevice(DPCTLSyclDeviceRef DRef): + """ + C-API function to create :class:`dpctl.SyclDevice` instance + from the given opaque device reference. + """ + cdef DPCTLSyclDeviceRef copied_DRef = DPCTLDevice_Copy(DRef) + return SyclDevice._create(copied_DRef) diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index f78e19c326..1d02450775 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -64,6 +64,15 @@ cdef api DPCTLSyclEventRef get_event_ref(SyclEvent ev): return ev.get_event_ref() +cdef api SyclEvent make_SyclEvent(DPCTLSyclEventRef ERef): + """ + C-API function to create :class:`dpctl.SyclEvent` + instance from opaque sycl event reference. + """ + cdef DPCTLSyclEventRef copied_ERef = DPCTLEvent_Copy(ERef) + return SyclEvent._create(copied_ERef) + + cdef void _event_capsule_deleter(object o): cdef DPCTLSyclEventRef ERef = NULL if pycapsule.PyCapsule_IsValid(o, "SyclEventRef"): diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 1091bbd765..614b7b2ae8 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -1007,3 +1007,12 @@ cdef api DPCTLSyclQueueRef get_queue_ref(SyclQueue q): :class:`dpctl.SyclQueue` instance. """ return q.get_queue_ref() + + +cdef api SyclQueue make_SyclQueue(DPCTLSyclQueueRef QRef): + """ + C-API function to create :class:`dpctl.SyclQueue` instance + from the given opaque queue reference. + """ + cdef DPCTLSyclQueueRef copied_QRef = DPCTLQueue_Copy(QRef) + return SyclQueue._create(copied_QRef) From 3691cfb7ad12670d2d6e4420b4f3b349f937d73c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 12:13:14 -0600 Subject: [PATCH 076/229] Reorg of dpctl/include folder 1. `dpctl/include` folder that is being installed in the site-packages/dpctl changes layout. It now has the structure dpctl/ include/ syclinterface/ *.h # all files that used to be in dpctl/include are moving here syclinterface.h dpctl_capi.h dpctl4pybind11.hpp _backend.pxd was modified to include syclinterface/ into path of header files from which we are importing build_backend.py was modified to copy install/include into dpctl/include/syclinterface instead of dpctl/include, and steps were added to also copy newly added dpctl/apis/include/* files into dpctl/include/ 2. Now, to work with syclinterface, one only needs `#include "syclinterface.h"` assuming `-I$(python -c "import dpctl; print(dpctl.get_include())")` is used 3. To work with `dpctl` objects using Python C-API one needs ``#include "dpctl_capi.h"`` where objects, types and C-API functions are declared. 4. To work with pybind11, one needs ``#include "dpctl4pybind11.hpp"`` where type casters are defined to map betwen ``sycl::queue`` and `dpctl.SyclQueue`, and correspondingly for ``sycl::device``, ``sycl::context`` and ``sycl::event``. Modifications to examples are forthcoming _backend.pxd should use syclinterface/ to reference .h files --- dpctl/_backend.pxd | 34 ++++----- dpctl/apis/include/dpctl4pybind11.hpp | 100 ++++++++++++++++++++++++++ dpctl/apis/include/dpctl_capi.h | 55 ++++++++++++++ dpctl/apis/include/syclinterface.h | 44 ++++++++++++ scripts/build_backend.py | 8 ++- setup.py | 2 +- 6 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 dpctl/apis/include/dpctl4pybind11.hpp create mode 100644 dpctl/apis/include/dpctl_capi.h create mode 100644 dpctl/apis/include/syclinterface.h diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index 8a74222b06..ab4af7d136 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -25,15 +25,15 @@ from libc.stdint cimport int64_t, uint32_t from libcpp cimport bool -cdef extern from "dpctl_error_handler_type.h": +cdef extern from "syclinterface/dpctl_error_handler_type.h": ctypedef void error_handler_callback(int err_code) -cdef extern from "dpctl_utils.h": +cdef extern from "syclinterface/dpctl_utils.h": cdef void DPCTLCString_Delete(const char *str) cdef void DPCTLSize_t_Array_Delete(size_t *arr) -cdef extern from "dpctl_sycl_enum_types.h": +cdef extern from "syclinterface/dpctl_sycl_enum_types.h": ctypedef enum _backend_type 'DPCTLSyclBackendType': _ALL_BACKENDS 'DPCTL_ALL_BACKENDS' _CUDA 'DPCTL_CUDA' @@ -111,7 +111,7 @@ cdef extern from "dpctl_sycl_enum_types.h": _COMPLETE 'DPCTL_COMPLETE' -cdef extern from "dpctl_sycl_types.h": +cdef extern from "syclinterface/dpctl_sycl_types.h": cdef struct DPCTLOpaqueSyclContext cdef struct DPCTLOpaqueSyclDevice cdef struct DPCTLOpaqueSyclDeviceSelector @@ -133,12 +133,12 @@ cdef extern from "dpctl_sycl_types.h": ctypedef DPCTLOpaqueSyclUSM *DPCTLSyclUSMRef -cdef extern from "dpctl_sycl_device_manager.h": +cdef extern from "syclinterface/dpctl_sycl_device_manager.h": cdef struct DPCTLDeviceVector ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef -cdef extern from "dpctl_sycl_device_interface.h": +cdef extern from "syclinterface/dpctl_sycl_device_interface.h": cdef bool DPCTLDevice_AreEq(const DPCTLSyclDeviceRef DRef1, const DPCTLSyclDeviceRef DRef2) cdef DPCTLSyclDeviceRef DPCTLDevice_Copy(const DPCTLSyclDeviceRef DRef) @@ -192,7 +192,7 @@ cdef extern from "dpctl_sycl_device_interface.h": cdef DPCTLSyclDeviceRef DPCTLDevice_GetParentDevice(const DPCTLSyclDeviceRef DRef) -cdef extern from "dpctl_sycl_device_manager.h": +cdef extern from "syclinterface/dpctl_sycl_device_manager.h": cdef DPCTLDeviceVectorRef DPCTLDeviceVector_CreateFromArray( size_t nelems, DPCTLSyclDeviceRef *elems) @@ -213,7 +213,7 @@ cdef extern from "dpctl_sycl_device_manager.h": cdef int64_t DPCTLDeviceMgr_GetRelativeId(const DPCTLSyclDeviceRef DRef) -cdef extern from "dpctl_sycl_device_selector_interface.h": +cdef extern from "syclinterface/dpctl_sycl_device_selector_interface.h": DPCTLSyclDeviceSelectorRef DPCTLAcceleratorSelector_Create() DPCTLSyclDeviceSelectorRef DPCTLDefaultSelector_Create() DPCTLSyclDeviceSelectorRef DPCTLCPUSelector_Create() @@ -224,7 +224,7 @@ cdef extern from "dpctl_sycl_device_selector_interface.h": int DPCTLDeviceSelector_Score(DPCTLSyclDeviceSelectorRef, DPCTLSyclDeviceRef) -cdef extern from "dpctl_sycl_event_interface.h": +cdef extern from "syclinterface/dpctl_sycl_event_interface.h": cdef DPCTLSyclEventRef DPCTLEvent_Create() cdef DPCTLSyclEventRef DPCTLEvent_Copy(const DPCTLSyclEventRef ERef) cdef void DPCTLEvent_Wait(DPCTLSyclEventRef ERef) @@ -246,13 +246,13 @@ cdef extern from "dpctl_sycl_event_interface.h": cdef size_t DPCTLEvent_GetProfilingInfoEnd(DPCTLSyclEventRef ERef) -cdef extern from "dpctl_sycl_kernel_interface.h": +cdef extern from "syclinterface/dpctl_sycl_kernel_interface.h": cdef const char* DPCTLKernel_GetFunctionName(const DPCTLSyclKernelRef KRef) cdef size_t DPCTLKernel_GetNumArgs(const DPCTLSyclKernelRef KRef) cdef void DPCTLKernel_Delete(DPCTLSyclKernelRef KRef) -cdef extern from "dpctl_sycl_platform_manager.h": +cdef extern from "syclinterface/dpctl_sycl_platform_manager.h": cdef struct DPCTLPlatformVector ctypedef DPCTLPlatformVector *DPCTLPlatformVectorRef @@ -265,7 +265,7 @@ cdef extern from "dpctl_sycl_platform_manager.h": cdef void DPCTLPlatformMgr_PrintInfo(const DPCTLSyclPlatformRef, size_t) -cdef extern from "dpctl_sycl_platform_interface.h": +cdef extern from "syclinterface/dpctl_sycl_platform_interface.h": cdef DPCTLSyclPlatformRef DPCTLPlatform_Copy(const DPCTLSyclPlatformRef) cdef DPCTLSyclPlatformRef DPCTLPlatform_Create() cdef DPCTLSyclPlatformRef DPCTLPlatform_CreateFromSelector( @@ -278,7 +278,7 @@ cdef extern from "dpctl_sycl_platform_interface.h": cdef DPCTLPlatformVectorRef DPCTLPlatform_GetPlatforms() -cdef extern from "dpctl_sycl_context_interface.h": +cdef extern from "syclinterface/dpctl_sycl_context_interface.h": cdef DPCTLSyclContextRef DPCTLContext_Create( const DPCTLSyclDeviceRef DRef, error_handler_callback *handler, @@ -299,7 +299,7 @@ cdef extern from "dpctl_sycl_context_interface.h": cdef void DPCTLContext_Delete(DPCTLSyclContextRef CtxRef) -cdef extern from "dpctl_sycl_program_interface.h": +cdef extern from "syclinterface/dpctl_sycl_program_interface.h": cdef DPCTLSyclProgramRef DPCTLProgram_CreateFromSpirv( const DPCTLSyclContextRef Ctx, const void *IL, @@ -317,7 +317,7 @@ cdef extern from "dpctl_sycl_program_interface.h": cdef void DPCTLProgram_Delete(DPCTLSyclProgramRef PRef) -cdef extern from "dpctl_sycl_queue_interface.h": +cdef extern from "syclinterface/dpctl_sycl_queue_interface.h": cdef bool DPCTLQueue_AreEq(const DPCTLSyclQueueRef QRef1, const DPCTLSyclQueueRef QRef2) cdef DPCTLSyclQueueRef DPCTLQueue_Create( @@ -381,7 +381,7 @@ cdef extern from "dpctl_sycl_queue_interface.h": cdef bool DPCTLQueue_HasEnableProfiling(const DPCTLSyclQueueRef QRef) -cdef extern from "dpctl_sycl_queue_manager.h": +cdef extern from "syclinterface/dpctl_sycl_queue_manager.h": cdef DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() cdef bool DPCTLQueueMgr_GlobalQueueIsCurrent() cdef bool DPCTLQueueMgr_IsCurrentQueue(const DPCTLSyclQueueRef QRef) @@ -391,7 +391,7 @@ cdef extern from "dpctl_sycl_queue_manager.h": cdef size_t DPCTLQueueMgr_GetQueueStackSize() -cdef extern from "dpctl_sycl_usm_interface.h": +cdef extern from "syclinterface/dpctl_sycl_usm_interface.h": cdef DPCTLSyclUSMRef DPCTLmalloc_shared( size_t size, DPCTLSyclQueueRef QRef) diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp new file mode 100644 index 0000000000..2d9775365d --- /dev/null +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -0,0 +1,100 @@ +//===----------- dpctl4pybind11.h - Headers for type pybind11 casters -*-C-*- +//===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines imports for dcptl's Python C-API +//===----------------------------------------------------------------------===// + +#pragma once + +#include "dpctl_capi.h" +#include +#include + +namespace py = pybind11; + +namespace pybind11 +{ +namespace detail +{ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::queue, _("dpctl.SyclQueue")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclQueueType)) { + DPCTLSyclQueueRef QRef = + get_queue_ref(reinterpret_cast(source)); + sycl::queue *q = reinterpret_cast(QRef); + value = *q; + return true; + } + else { + throw std::runtime_error( + "Input is of unexpected type, expected egapi.Example"); + } + } + + static handle cast(sycl::queue src, return_value_policy, handle) + { + auto tmp = make_SyclQueue(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; +} // namespace detail +} // namespace pybind11 + +namespace pybind11 +{ +namespace detail +{ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::device, _("dpctl.SyclDevice")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclDeviceType)) { + DPCTLSyclDeviceRef DRef = + get_device_ref(reinterpret_cast(source)); + sycl::device *d = reinterpret_cast(DRef); + value = *d; + return true; + } + else { + throw std::runtime_error( + "Input is of unexpected type, expected egapi.Example"); + } + } + + static handle cast(sycl::device src, return_value_policy, handle) + { + auto tmp = make_SyclDevice(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; +} // namespace detail +} // namespace pybind11 diff --git a/dpctl/apis/include/dpctl_capi.h b/dpctl/apis/include/dpctl_capi.h new file mode 100644 index 0000000000..0a484bcff7 --- /dev/null +++ b/dpctl/apis/include/dpctl_capi.h @@ -0,0 +1,55 @@ +//===----------- dpctl_capi.h - Headers for dpctl's C-API -*-C-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines imports for dcptl's Python C-API +//===----------------------------------------------------------------------===// + +#pragma once + +// clang-format off +// Ordering of includes is important here. dpctl_sycl_types defines types +// used by dpctl's Python C-API headers. +#include "syclinterface/dpctl_sycl_types.h" +#include "../_sycl_device.h" +#include "../_sycl_device_api.h" +#include "../_sycl_context.h" +#include "../_sycl_context_api.h" +#include "../_sycl_event.h" +#include "../_sycl_event_api.h" +#include "../_sycl_queue.h" +#include "../_sycl_queue_api.h" +// clang-format on + +/* + * Function to import dpctl and make C-API functions available. + * C functions can use dpctl's C-API functions without linking to + * shared objects defining this symbols, if they call `import_dpctl()` + * prior to using those symbols. + */ +void import_dpctl(void) +{ + import_dpctl___sycl_device(); + import_dpctl___sycl_context(); + import_dpctl___sycl_event(); + import_dpctl___sycl_queue(); + + return; +} diff --git a/dpctl/apis/include/syclinterface.h b/dpctl/apis/include/syclinterface.h new file mode 100644 index 0000000000..04ab1cfbbb --- /dev/null +++ b/dpctl/apis/include/syclinterface.h @@ -0,0 +1,44 @@ +//=== syclinterace.h - single include header for libsyclinterface -*-C-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file includes all the headers of syclinterface/ +//===----------------------------------------------------------------------===// + +#pragma once + +// clang-format off +#include "syclinterface/dpctl_sycl_types.h" +#include "syclinterface/dpctl_sycl_enum_types.h" +#include "syclinterface/dpctl_service.h" +#include "syclinterface/dpctl_vector.h" +#include "syclinterface/dpctl_utils.h" +#include "syclinterface/dpctl_sycl_device_selector_interface.h" +#include "syclinterface/dpctl_sycl_context_interface.h" +#include "syclinterface/dpctl_sycl_device_interface.h" +#include "syclinterface/dpctl_sycl_event_interface.h" +#include "syclinterface/dpctl_sycl_platform_interface.h" +#include "syclinterface/dpctl_sycl_queue_interface.h" +#include "syclinterface/dpctl_sycl_usm_interface.h" + +#include "syclinterface/dpctl_sycl_device_manager.h" +#include "syclinterface/dpctl_sycl_platform_manager.h" +#include "syclinterface/dpctl_sycl_queue_manager.h" +// clang-format on diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 0ee4e8799a..d188dfb8b3 100644 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -213,9 +213,15 @@ def build_backend( shutil.rmtree(include_dir) shutil.copytree( - os.path.join(dpctl_dir, "libsyclinterface", "include"), include_dir + os.path.join(dpctl_dir, "libsyclinterface", "include"), + os.path.join(include_dir, "syclinterface"), ) + for file in glob.glob( + os.path.join(dpctl_dir, "dpctl", "apis", "include", "*.h*") + ): + shutil.copy(file, include_dir) + if __name__ == "__main__": build_backend() diff --git a/setup.py b/setup.py index c690c9ca28..b624ae473a 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ # to build_ext command _coverage = False dpctl_sycl_interface_lib = "dpctl" -dpctl_sycl_interface_include = r"dpctl/include" +dpctl_sycl_interface_include = os.path.join("dpctl", "include") # Get long description with open("README.md", "r", encoding="utf-8") as file: From ca3ee4d2ec29a7c1c474c5152970fd5f17d48fcf Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 13:14:04 -0600 Subject: [PATCH 077/229] Streamlined pybind11 examples using dpctl4pybind11.hpp --- .../_usm_alloc_example.cpp | 25 +-- .../pybind11/use_dpctl_syclqueue/_example.cpp | 105 ++++++++++++ .../pybind11/use_dpctl_syclqueue/example.py | 2 +- .../use_dpctl_syclqueue/pybind11_example.cpp | 159 ------------------ .../pybind11/use_dpctl_syclqueue/setup.py | 4 +- 5 files changed, 113 insertions(+), 182 deletions(-) create mode 100644 examples/pybind11/use_dpctl_syclqueue/_example.cpp delete mode 100644 examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp diff --git a/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp b/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp index 7a4e846f6d..30216dac55 100644 --- a/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp +++ b/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp @@ -33,12 +33,7 @@ //===----------------------------------------------------------------------===// #include -// clang-format off -#include "dpctl_sycl_types.h" -#include "../_sycl_queue.h" -#include "../_sycl_queue_api.h" -// clang-format on - +#include "dpctl4pybind11.hpp" #include "pybind11/pybind11.h" #include "pybind11/stl.h" @@ -87,19 +82,9 @@ struct DMatrix vec_t vec_; }; -DMatrix create_matrix(py::object queue, size_t n, size_t m) +DMatrix create_matrix(sycl::queue &q, size_t n, size_t m) { - PyObject *queue_ptr = queue.ptr(); - if (PyObject_TypeCheck(queue_ptr, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_ptr)); - sycl::queue *q = reinterpret_cast(QRef); - - return DMatrix(*q, n, m); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } + return DMatrix(q, n, m); } py::dict construct_sua_iface(DMatrix &m) @@ -149,8 +134,8 @@ py::list tolist(DMatrix &m) PYBIND11_MODULE(external_usm_alloc, m) { - // Import the dpctl._sycl_queue extension - import_dpctl___sycl_queue(); + // Import the dpctl extensions + import_dpctl(); py::class_ dm(m, "DMatrix"); dm.def(py::init(&create_matrix), diff --git a/examples/pybind11/use_dpctl_syclqueue/_example.cpp b/examples/pybind11/use_dpctl_syclqueue/_example.cpp new file mode 100644 index 0000000000..7b26a35858 --- /dev/null +++ b/examples/pybind11/use_dpctl_syclqueue/_example.cpp @@ -0,0 +1,105 @@ +//==- pybind11_example.cpp - Example of Pybind11 extension working with -===// +// dpctl Python objects. +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements Pybind11-generated extension exposing functions that +/// take dpctl Python objects, such as dpctl.SyclQueue, dpctl.SyclDevice as +/// arguments. +/// +//===----------------------------------------------------------------------===// + +#include "dpctl4pybind11.hpp" +#include +#include +#include +#include + +namespace py = pybind11; + +uint64_t get_device_global_mem_size(sycl::device &d) +{ + return d.get_info(); +} + +uint64_t get_device_local_mem_size(sycl::device &d) +{ + return d.get_info(); +} + +py::array_t +offloaded_array_mod(sycl::queue &q, + py::array_t array, + int64_t mod) +{ + py::buffer_info arg_pybuf = array.request(); + if (arg_pybuf.ndim != 1) { + throw std::runtime_error("Expecting a vector"); + } + if (mod <= 0) { + throw std::runtime_error("Modulus must be non-negative"); + } + + size_t n = arg_pybuf.size; + + auto res = py::array_t(n); + py::buffer_info res_pybuf = res.request(); + + int64_t *a = static_cast(arg_pybuf.ptr); + int64_t *r = static_cast(res_pybuf.ptr); + + { + const sycl::property_list props = { + sycl::property::buffer::use_host_ptr()}; + sycl::buffer a_buf(a, sycl::range<1>(n), props); + sycl::buffer r_buf(r, sycl::range<1>(n), props); + + q.submit([&](sycl::handler &cgh) { + sycl::accessor a_acc(a_buf, cgh, sycl::read_only); + sycl::accessor r_acc(r_buf, cgh, sycl::write_only, sycl::no_init); + + cgh.parallel_for(sycl::range<1>(n), [=](sycl::id<1> idx) { + r_acc[idx] = a_acc[idx] % mod; + }); + }).wait_and_throw(); + } + + return res; +} + +PYBIND11_MODULE(use_queue_device_ext, m) +{ + // Import the dpctl extensions + import_dpctl(); + m.def( + "get_max_compute_units", + [=](sycl::queue &q) -> size_t { + return q.get_device() + .get_info(); + }, + "Computes max_compute_units property of the device underlying given " + "dpctl.SyclQueue"); + m.def("get_device_global_mem_size", &get_device_global_mem_size, + "Computes amount of global memory of the given dpctl.SyclDevice"); + m.def("get_device_local_mem_size", &get_device_local_mem_size, + "Computes amount of local memory of the given dpctl.SyclDevice"); + m.def("offloaded_array_mod", &offloaded_array_mod, + "Compute offloaded modular reduction of integer-valued NumPy array"); +} diff --git a/examples/pybind11/use_dpctl_syclqueue/example.py b/examples/pybind11/use_dpctl_syclqueue/example.py index 4c53bfafe2..1980ca82d3 100644 --- a/examples/pybind11/use_dpctl_syclqueue/example.py +++ b/examples/pybind11/use_dpctl_syclqueue/example.py @@ -17,7 +17,7 @@ # coding: utf-8 import numpy as np -import pybind11_example as eg +import use_queue_device_ext as eg import dpctl diff --git a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp b/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp deleted file mode 100644 index ed22daec2a..0000000000 --- a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp +++ /dev/null @@ -1,159 +0,0 @@ -//==- pybind11_example.cpp - Example of Pybind11 extension working with -===// -// dpctl Python objects. -// -// Data Parallel Control (dpctl) -// -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// This file implements Pybind11-generated extension exposing functions that -/// take dpctl Python objects, such as dpctl.SyclQueue, dpctl.SyclDevice as -/// arguments. -/// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include - -// clang-format off -// Ordering of includes is important here. dpctl_sycl_types defines types -// used by dpctl's Python C-API headers. -#include "dpctl_sycl_types.h" -#include "../_sycl_queue.h" -#include "../_sycl_queue_api.h" -#include "../_sycl_device.h" -#include "../_sycl_device_api.h" -// clang-format on - -namespace py = pybind11; - -size_t get_max_compute_units(py::object queue) -{ - PyObject *queue_ptr = queue.ptr(); - if (PyObject_TypeCheck(queue_ptr, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_ptr)); - sycl::queue *q = reinterpret_cast(QRef); - - return q->get_device() - .get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } -} - -uint64_t get_device_global_mem_size(py::object device) -{ - PyObject *device_pycapi = device.ptr(); - if (PyObject_TypeCheck(device_pycapi, &PySyclDeviceType)) { - DPCTLSyclDeviceRef DRef = get_device_ref( - reinterpret_cast(device_pycapi)); - sycl::device *d_ptr = reinterpret_cast(DRef); - return d_ptr->get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclDevice as argument"); - } -} - -uint64_t get_device_local_mem_size(py::object device) -{ - PyObject *device_pycapi = device.ptr(); - if (PyObject_TypeCheck(device_pycapi, &PySyclDeviceType)) { - DPCTLSyclDeviceRef DRef = get_device_ref( - reinterpret_cast(device_pycapi)); - sycl::device *d_ptr = reinterpret_cast(DRef); - return d_ptr->get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclDevice as argument"); - } -} - -py::array_t -offloaded_array_mod(py::object queue, - py::array_t array, - int64_t mod) -{ - sycl::queue *q_ptr; - - PyObject *queue_pycapi = queue.ptr(); - if (PyObject_TypeCheck(queue_pycapi, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_pycapi)); - q_ptr = reinterpret_cast(QRef); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } - - py::buffer_info arg_pybuf = array.request(); - if (arg_pybuf.ndim != 1) { - throw std::runtime_error("Expecting a vector"); - } - if (mod <= 0) { - throw std::runtime_error("Modulus must be non-negative"); - } - - size_t n = arg_pybuf.size; - - auto res = py::array_t(n); - py::buffer_info res_pybuf = res.request(); - - int64_t *a = static_cast(arg_pybuf.ptr); - int64_t *r = static_cast(res_pybuf.ptr); - - { - const sycl::property_list props = { - sycl::property::buffer::use_host_ptr()}; - sycl::buffer a_buf(a, sycl::range<1>(n), props); - sycl::buffer r_buf(r, sycl::range<1>(n), props); - - q_ptr - ->submit([&](sycl::handler &cgh) { - sycl::accessor a_acc(a_buf, cgh, sycl::read_only); - sycl::accessor r_acc(r_buf, cgh, sycl::write_only, - sycl::no_init); - - cgh.parallel_for(sycl::range<1>(n), [=](sycl::id<1> idx) { - r_acc[idx] = a_acc[idx] % mod; - }); - }) - .wait_and_throw(); - } - - return res; -} - -PYBIND11_MODULE(pybind11_example, m) -{ - // Import the dpctl._sycl_queue, dpctl._sycl_device extensions - import_dpctl___sycl_device(); - import_dpctl___sycl_queue(); - m.def("get_max_compute_units", &get_max_compute_units, - "Computes max_compute_units property of the device underlying given " - "dpctl.SyclQueue"); - m.def("get_device_global_mem_size", &get_device_global_mem_size, - "Computes amount of global memory of the given dpctl.SyclDevice"); - m.def("get_device_local_mem_size", &get_device_local_mem_size, - "Computes amount of local memory of the given dpctl.SyclDevice"); - m.def("offloaded_array_mod", &offloaded_array_mod, - "Compute offloaded modular reduction of integer-valued NumPy array"); -} diff --git a/examples/pybind11/use_dpctl_syclqueue/setup.py b/examples/pybind11/use_dpctl_syclqueue/setup.py index 4569c99029..34eeebe38c 100644 --- a/examples/pybind11/use_dpctl_syclqueue/setup.py +++ b/examples/pybind11/use_dpctl_syclqueue/setup.py @@ -21,8 +21,8 @@ exts = [ Pybind11Extension( - "pybind11_example", - ["./pybind11_example.cpp"], + "use_queue_device_ext", + ["./_example.cpp"], include_dirs=[dpctl.get_include()], extra_compile_args=["-fPIC"], extra_link_args=["-fPIC"], From fd35965c1bed704e33d086be5f91bd3c8a588ab1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 13:43:14 -0600 Subject: [PATCH 078/229] Changed include of dpctl_sycl_types.h to include of syclinterface.h --- examples/cython/sycl_buffer/use_sycl_buffer.cpp | 2 +- examples/cython/sycl_buffer/use_sycl_buffer.h | 2 +- examples/cython/usm_memory/sycl_blackscholes.cpp | 2 +- examples/cython/usm_memory/sycl_blackscholes.hpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.cpp b/examples/cython/sycl_buffer/use_sycl_buffer.cpp index 57b28de1fb..afbc814aa9 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.cpp +++ b/examples/cython/sycl_buffer/use_sycl_buffer.cpp @@ -28,7 +28,7 @@ //===----------------------------------------------------------------------===// #include "use_sycl_buffer.h" -#include "dpctl_sycl_types.h" +#include "syclinterface.h" #include #include diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.h b/examples/cython/sycl_buffer/use_sycl_buffer.h index 66bfe756bc..9697f81fd2 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.h +++ b/examples/cython/sycl_buffer/use_sycl_buffer.h @@ -1,4 +1,4 @@ -#include "dpctl_sycl_types.h" +#include "syclinterface.h" #include extern int c_columnwise_total(DPCTLSyclQueueRef q, diff --git a/examples/cython/usm_memory/sycl_blackscholes.cpp b/examples/cython/usm_memory/sycl_blackscholes.cpp index 67406345b6..87c1c517ce 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.cpp +++ b/examples/cython/usm_memory/sycl_blackscholes.cpp @@ -27,7 +27,7 @@ //===----------------------------------------------------------------------===// #include "sycl_blackscholes.hpp" -#include "dpctl_sycl_types.h" +#include "syclinterface.h" #include #include #include diff --git a/examples/cython/usm_memory/sycl_blackscholes.hpp b/examples/cython/usm_memory/sycl_blackscholes.hpp index 02181e2756..e61b9b47f6 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.hpp +++ b/examples/cython/usm_memory/sycl_blackscholes.hpp @@ -24,7 +24,7 @@ /// //===----------------------------------------------------------------------===// -#include "dpctl_sycl_types.h" +#include "syclinterface.h" #include template From 3b18c73e0ba39754d500c3fdf035604fde618c63 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 14:03:21 -0600 Subject: [PATCH 079/229] Make sure to include dpctl4pybind11.hpp in the manifest --- MANIFEST.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5ce66287a8..604da428cc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include versioneer.py recursive-include dpctl/include *.h -recursive-include dpctl/tensor/include * +include dpctl/include/dpctl4pybind11.hpp recursive-include dpctl *.pxd include dpctl/_sycl_context.h include dpctl/_sycl_context_api.h @@ -14,5 +14,6 @@ include dpctl/memory/_memory.h include dpctl/memory/_memory_api.h include dpctl/tensor/_usmarray.h include dpctl/tensor/_usmarray_api.h +recursive-include dpctl/tensor/include * include dpctl/tests/input_files/* include dpctl/tests/*.pyx From 5f040b5554929e4ea15882a604bd972fe30e4f29 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 14:52:48 -0600 Subject: [PATCH 080/229] Added tests to make_Sycl* C-API functions --- dpctl/tests/test_sycl_context.py | 24 ++++++++++++++++- dpctl/tests/test_sycl_device.py | 45 ++++++++++++++++++++++++++++++++ dpctl/tests/test_sycl_event.py | 24 ++++++++++++++++- dpctl/tests/test_sycl_queue.py | 24 +++++++++++++++-- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/dpctl/tests/test_sycl_context.py b/dpctl/tests/test_sycl_context.py index 99d72c14eb..1b67d1f4f3 100644 --- a/dpctl/tests/test_sycl_context.py +++ b/dpctl/tests/test_sycl_context.py @@ -190,7 +190,7 @@ def test_context_repr(): assert type(ctx.__repr__()) is str -def test_cpython_api(): +def test_cpython_api_get_context_ref(): import ctypes import sys @@ -213,6 +213,28 @@ def test_cpython_api(): assert r1 == r2 +def test_cpython_api_make_SyclContext(): + import ctypes + import sys + + ctx = dpctl.SyclContext() + mod = sys.modules[ctx.__class__.__module__] + # get capsule storign make_SyclContext function ptr + make_ctx_fn_cap = mod.__pyx_capi__["make_SyclContext"] + # construct Python callable to invoke "make_SyclContext" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_ctx_fn_ptr = cap_ptr_fn( + make_ctx_fn_cap, b"struct PySyclContextObject *(DPCTLSyclContextRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_ctx_fn = callable_maker(make_ctx_fn_ptr) + + ctx2 = make_ctx_fn(ctx.addressof_ref()) + assert ctx == ctx2 + + def test_invalid_capsule(): cap = create_invalid_capsule() with pytest.raises(ValueError): diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 5b3f30c89d..329444ac90 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -729,3 +729,48 @@ def test_handle_no_device(): dpctl.select_device_with_aspects(["gpu", "cpu"]) with pytest.raises(ValueError): dpctl.select_device_with_aspects("cpu", excluded_aspects="cpu") + + +def test_cpython_api_get_device_ref(): + import ctypes + import sys + + d = dpctl.SyclDevice() + mod = sys.modules[d.__class__.__module__] + # get capsule storign get_device_ref function ptr + d_ref_fn_cap = mod.__pyx_capi__["get_device_ref"] + # construct Python callable to invoke "get_device_ref" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + d_ref_fn_ptr = cap_ptr_fn( + d_ref_fn_cap, b"DPCTLSyclDeviceRef (struct PySyclDeviceObject *)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object) + get_device_ref_fn = callable_maker(d_ref_fn_ptr) + + r2 = d.addressof_ref() + r1 = get_device_ref_fn(d) + assert r1 == r2 + + +def test_cpython_api_make_SyclDevice(): + import ctypes + import sys + + d = dpctl.SyclDevice() + mod = sys.modules[d.__class__.__module__] + # get capsule storign make_SyclContext function ptr + make_d_fn_cap = mod.__pyx_capi__["make_SyclDevice"] + # construct Python callable to invoke "make_SyclDevice" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_d_fn_ptr = cap_ptr_fn( + make_d_fn_cap, b"struct PySyclDeviceObject *(DPCTLSyclDeviceRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_d_fn = callable_maker(make_d_fn_ptr) + + d2 = make_d_fn(d.addressof_ref()) + assert d == d2 diff --git a/dpctl/tests/test_sycl_event.py b/dpctl/tests/test_sycl_event.py index 24c4aa4b75..12c48e48c7 100644 --- a/dpctl/tests/test_sycl_event.py +++ b/dpctl/tests/test_sycl_event.py @@ -234,7 +234,7 @@ def test_addressof_ref(): assert type(ref) is int -def test_cpython_api(): +def test_cpython_api_get_event_ref(): import ctypes import sys @@ -255,3 +255,25 @@ def test_cpython_api(): r2 = ev.addressof_ref() r1 = get_event_ref_fn(ev) assert r1 == r2 + + +def test_cpython_api_make_SyclEvent(): + import ctypes + import sys + + ev = dpctl.SyclEvent() + mod = sys.modules[ev.__class__.__module__] + # get capsule storing make_SyclEvent function ptr + make_e_fn_cap = mod.__pyx_capi__["make_SyclEvent"] + # construct Python callable to invoke "make_SyclDevice" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_e_fn_ptr = cap_ptr_fn( + make_e_fn_cap, b"struct PySyclEventObject *(DPCTLSyclEventRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_e_fn = callable_maker(make_e_fn_ptr) + + ev2 = make_e_fn(ev.addressof_ref()) + assert type(ev) == type(ev2) diff --git a/dpctl/tests/test_sycl_queue.py b/dpctl/tests/test_sycl_queue.py index 0f36c319b2..6ad2571968 100644 --- a/dpctl/tests/test_sycl_queue.py +++ b/dpctl/tests/test_sycl_queue.py @@ -469,10 +469,10 @@ def test_queue_capsule(): assert q2 != [] # compare with other types -def test_cpython_api(): +def test_cpython_api_get_queue_ref(): q = dpctl.SyclQueue() mod = sys.modules[q.__class__.__module__] - # get capsule storign get_context_ref function ptr + # get capsule storign get_queue_ref function ptr q_ref_fn_cap = mod.__pyx_capi__["get_queue_ref"] # construct Python callable to invoke "get_queue_ref" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer @@ -489,6 +489,26 @@ def test_cpython_api(): assert r1 == r2 +def test_cpython_api_make_SyclQueue(): + q = dpctl.SyclQueue() + mod = sys.modules[q.__class__.__module__] + # get capsule storing make_SyclQueue function ptr + make_SyclQueue_fn_cap = mod.__pyx_capi__["make_SyclQueue"] + # construct Python callable to invoke "make_SyclQueue" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_SyclQueue_fn_ptr = cap_ptr_fn( + make_SyclQueue_fn_cap, b"struct PySyclQueueObject *(DPCTLSyclQueueRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_SyclQueue_fn = callable_maker(make_SyclQueue_fn_ptr) + + q2 = make_SyclQueue_fn(q.addressof_ref()) + assert q.sycl_device == q2.sycl_device + assert q.sycl_context == q2.sycl_context + + def test_constructor_many_arg(): with pytest.raises(TypeError): dpctl.SyclQueue(None, None, None, None) From 0e38e21047920bd401e5d51efa922a3c68093065 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 14:59:47 -0600 Subject: [PATCH 081/229] Added documenting C++ comment Fixed runtime_error verbiage. --- dpctl/apis/include/dpctl4pybind11.hpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp index 2d9775365d..b959c66c59 100644 --- a/dpctl/apis/include/dpctl4pybind11.hpp +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -35,6 +35,11 @@ namespace pybind11 { namespace detail { + +/* This type caster associates ``sycl::queue`` C++ class with + * :class:`dpctl.SyclQueue` for the purposes of generation of + * Python bindings by pybind11. + */ template <> struct type_caster { public: @@ -52,7 +57,7 @@ template <> struct type_caster } else { throw std::runtime_error( - "Input is of unexpected type, expected egapi.Example"); + "Input is of unexpected type, expected dpctl.SyclQueue"); } } @@ -69,6 +74,10 @@ namespace pybind11 { namespace detail { +/* This type caster associates ``sycl::device`` C++ class with + * :class:`dpctl.SyclDevice` for the purposes of generation of + * Python bindings by pybind11. + */ template <> struct type_caster { public: @@ -86,7 +95,7 @@ template <> struct type_caster } else { throw std::runtime_error( - "Input is of unexpected type, expected egapi.Example"); + "Input is of unexpected type, expected dpctl.SyclDevice"); } } From e88014310b111a4d679c08cdbf53189f47a8a061 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 15:06:22 -0600 Subject: [PATCH 082/229] Per PR feedback: dpctl/include/syclinterface.h -> dpctl/include/dpctl_sycl_interface.h --- dpctl/apis/include/{syclinterface.h => dpctl_sycl_interface.h} | 1 - examples/cython/sycl_buffer/use_sycl_buffer.cpp | 2 +- examples/cython/sycl_buffer/use_sycl_buffer.h | 2 +- examples/cython/usm_memory/sycl_blackscholes.cpp | 2 +- examples/cython/usm_memory/sycl_blackscholes.hpp | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) rename dpctl/apis/include/{syclinterface.h => dpctl_sycl_interface.h} (99%) diff --git a/dpctl/apis/include/syclinterface.h b/dpctl/apis/include/dpctl_sycl_interface.h similarity index 99% rename from dpctl/apis/include/syclinterface.h rename to dpctl/apis/include/dpctl_sycl_interface.h index 04ab1cfbbb..4f4f4c2b19 100644 --- a/dpctl/apis/include/syclinterface.h +++ b/dpctl/apis/include/dpctl_sycl_interface.h @@ -37,7 +37,6 @@ #include "syclinterface/dpctl_sycl_platform_interface.h" #include "syclinterface/dpctl_sycl_queue_interface.h" #include "syclinterface/dpctl_sycl_usm_interface.h" - #include "syclinterface/dpctl_sycl_device_manager.h" #include "syclinterface/dpctl_sycl_platform_manager.h" #include "syclinterface/dpctl_sycl_queue_manager.h" diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.cpp b/examples/cython/sycl_buffer/use_sycl_buffer.cpp index afbc814aa9..ba85510d09 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.cpp +++ b/examples/cython/sycl_buffer/use_sycl_buffer.cpp @@ -28,7 +28,7 @@ //===----------------------------------------------------------------------===// #include "use_sycl_buffer.h" -#include "syclinterface.h" +#include "dpctl_sycl_interface.h" #include #include diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.h b/examples/cython/sycl_buffer/use_sycl_buffer.h index 9697f81fd2..d9ea64c993 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.h +++ b/examples/cython/sycl_buffer/use_sycl_buffer.h @@ -1,4 +1,4 @@ -#include "syclinterface.h" +#include "dpctl_sycl_interface.h" #include extern int c_columnwise_total(DPCTLSyclQueueRef q, diff --git a/examples/cython/usm_memory/sycl_blackscholes.cpp b/examples/cython/usm_memory/sycl_blackscholes.cpp index 87c1c517ce..8d6d0d6f85 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.cpp +++ b/examples/cython/usm_memory/sycl_blackscholes.cpp @@ -27,7 +27,7 @@ //===----------------------------------------------------------------------===// #include "sycl_blackscholes.hpp" -#include "syclinterface.h" +#include "dpctl_sycl_interface.h" #include #include #include diff --git a/examples/cython/usm_memory/sycl_blackscholes.hpp b/examples/cython/usm_memory/sycl_blackscholes.hpp index e61b9b47f6..1c38ebef4b 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.hpp +++ b/examples/cython/usm_memory/sycl_blackscholes.hpp @@ -24,7 +24,7 @@ /// //===----------------------------------------------------------------------===// -#include "syclinterface.h" +#include "dpctl_sycl_interface.h" #include template From d42e403c0771388776c2b65c6681d9209179da5e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 23 Nov 2021 15:18:37 -0600 Subject: [PATCH 083/229] Added type caster for event and context --- dpctl/apis/include/dpctl4pybind11.hpp | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp index b959c66c59..4182a2d9bb 100644 --- a/dpctl/apis/include/dpctl4pybind11.hpp +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -107,3 +107,80 @@ template <> struct type_caster }; } // namespace detail } // namespace pybind11 + +namespace pybind11 +{ +namespace detail +{ +/* This type caster associates ``sycl::context`` C++ class with + * :class:`dpctl.SyclContext` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::context, _("dpctl.SyclContext")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclContextType)) { + DPCTLSyclContextRef CRef = get_context_ref( + reinterpret_cast(source)); + sycl::context *ctx = reinterpret_cast(CRef); + value = *ctx; + return true; + } + else { + throw std::runtime_error( + "Input is of unexpected type, expected dpctl.SyclContext"); + } + } + + static handle cast(sycl::context src, return_value_policy, handle) + { + auto tmp = + make_SyclContext(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; +} // namespace detail +} // namespace pybind11 + +namespace pybind11 +{ +namespace detail +{ +/* This type caster associates ``sycl::event`` C++ class with + * :class:`dpctl.SyclEvent` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::event, _("dpctl.SyclEvent")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclEventType)) { + DPCTLSyclEventRef ERef = + get_event_ref(reinterpret_cast(source)); + sycl::event *ev = reinterpret_cast(ERef); + value = *ev; + return true; + } + else { + throw std::runtime_error( + "Input is of unexpected type, expected dpctl.SyclEvent"); + } + } + + static handle cast(sycl::event src, return_value_policy, handle) + { + auto tmp = make_SyclEvent(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; +} // namespace detail +} // namespace pybind11 From 22e31238ec1757e3d2e064782766f4ecb62bd643 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 2 Dec 2021 12:39:37 -0600 Subject: [PATCH 084/229] Merge pull request #703 from IntelPython/utils-license-and-docstring Added license header and docstring to dpctl.utils.__init__ --- dpctl/utils/__init__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dpctl/utils/__init__.py b/dpctl/utils/__init__.py index f45afa0fab..d4003fa825 100644 --- a/dpctl/utils/__init__.py +++ b/dpctl/utils/__init__.py @@ -1,3 +1,23 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Collection of utilities. +""" + from ._compute_follows_data import get_execution_queue __all__ = [ From 431bed8f7a55d89a7058ff53999db93c18378c94 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 3 Dec 2021 13:27:33 -0600 Subject: [PATCH 085/229] Make sure to fail worfklow step in doc generation of make Sphinx errs (#707) * Make sure to fail worfklow step in doc generation of make Sphinx errors out * Update .github/workflows/generate-docs.yml --- .github/workflows/generate-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index f5b780b0bd..69bd24eaff 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -75,7 +75,7 @@ jobs: cmake .. -DDPCTL_USE_MULTIVERSION_TEMPLATE=ON \ -DDPCTL_ENABLE_DOXYREST=ON \ -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 - make Sphinx + make Sphinx || exit 1 cd .. mv generated_docs/docs ~/docs git clean -dfx @@ -90,7 +90,7 @@ jobs: echo `pwd` cd master git rm -rf * - mv ~/docs/* . + mv ~/docs/* . || exit 1 git add . git config --global user.name 'github-actions[doc-deploy-bot]' git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' From 587470020c1194925e3e0895405f5dc9a46473c4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 3 Dec 2021 16:52:02 -0600 Subject: [PATCH 086/229] Exported dpctl.tensor.Device as oneAPI device class This is returned by X.device, as before. Addressed `dpt.asnumpy.__name__` not being 'anumpy', and `dpt.to_numpy.__name__` not being `to_numpy'. Added `normalize_queue_device` to `_device.py` as a utility --- dpctl/tensor/__init__.py | 7 ++-- dpctl/tensor/_copy_utils.py | 74 +++++++++++++++++++++++++++++++------ dpctl/tensor/_ctors.py | 34 ++++------------- dpctl/tensor/_device.py | 51 +++++++++++++++++++++++++ dpctl/tensor/_usmarray.pyx | 8 ++-- 5 files changed, 127 insertions(+), 47 deletions(-) diff --git a/dpctl/tensor/__init__.py b/dpctl/tensor/__init__.py index ab2bf72ebb..944c80fa47 100644 --- a/dpctl/tensor/__init__.py +++ b/dpctl/tensor/__init__.py @@ -29,16 +29,15 @@ """ -from dpctl.tensor._copy_utils import astype, copy -from dpctl.tensor._copy_utils import copy_from_numpy as from_numpy -from dpctl.tensor._copy_utils import copy_to_numpy as asnumpy -from dpctl.tensor._copy_utils import copy_to_numpy as to_numpy +from dpctl.tensor._copy_utils import asnumpy, astype, copy, from_numpy, to_numpy from dpctl.tensor._ctors import asarray, empty +from dpctl.tensor._device import Device from dpctl.tensor._dlpack import from_dlpack from dpctl.tensor._reshape import reshape from dpctl.tensor._usmarray import usm_ndarray __all__ = [ + "Device", "usm_ndarray", "asarray", "astype", diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index 9422210a6d..a158912d59 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -19,6 +19,7 @@ import dpctl.memory as dpm import dpctl.tensor as dpt +from dpctl.tensor._device import normalize_queue_device def contract_iter2(shape, strides1, strides2): @@ -64,7 +65,7 @@ def contract_iter2(shape, strides1, strides2): return (sh, st1, disp1, st2, disp2) -def has_memory_overlap(x1, x2): +def _has_memory_overlap(x1, x2): m1 = dpm.as_usm_memory(x1) m2 = dpm.as_usm_memory(x2) if m1.sycl_device == m2.sycl_device: @@ -77,7 +78,7 @@ def has_memory_overlap(x1, x2): return False -def copy_to_numpy(ary): +def _copy_to_numpy(ary): if type(ary) is not dpt.usm_ndarray: raise TypeError h = ary.usm_data.copy_to_host().view(ary.dtype) @@ -93,7 +94,7 @@ def copy_to_numpy(ary): ) -def copy_from_numpy(np_ary, usm_type="device", sycl_queue=None): +def _copy_from_numpy(np_ary, usm_type="device", sycl_queue=None): "Copies numpy array `np_ary` into a new usm_ndarray" # This may peform a copy to meet stated requirements Xnp = np.require(np_ary, requirements=["A", "O", "C", "E"]) @@ -111,7 +112,8 @@ def copy_from_numpy(np_ary, usm_type="device", sycl_queue=None): return Xusm -def copy_from_numpy_into(dst, np_ary): +def _copy_from_numpy_into(dst, np_ary): + "Copies `np_ary` into `dst` of type :class:`dpctl.tensor.usm_ndarray" if not isinstance(np_ary, np.ndarray): raise TypeError("Expected numpy.ndarray, got {}".format(type(np_ary))) src_ary = np.broadcast_to(np.asarray(np_ary, dtype=dst.dtype), dst.shape) @@ -122,6 +124,54 @@ def copy_from_numpy_into(dst, np_ary): usm_mem.copy_from_host(host_buf) +def from_numpy(np_ary, device=None, usm_type="device", sycl_queue=None): + """ + from_numpy(arg, device=None, usm_type="device", sycl_queue=None) + + Creates :class:`dpctl.tensor.usm_ndarray` from instance of + `numpy.ndarray`. + + Args: + arg: An instance of `numpy.ndarray` + device: array API specification of device where the output array + is created. + sycl_queue: a :class:`dpctl.SyclQueue` used to create the output + array is created + """ + q = normalize_queue_device(sycl_queue=sycl_queue, device=device) + return _copy_from_numpy(np_ary, usm_type=usm_type, sycl_queue=q) + + +def to_numpy(usm_ary): + """ + to_numpy(usm_ary) + + Copies content of :class:`dpctl.tensor.usm_ndarray` instance `usm_ary` + into `numpy.ndarray` instance of the same shape and same data type. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + An instance of `numpy.ndarray` populated with content of `usm_ary`. + """ + return _copy_to_numpy(usm_ary) + + +def asnumpy(usm_ary): + """ + asnumpy(usm_ary) + + Copies content of :class:`dpctl.tensor.usm_ndarray` instance `usm_ary` + into `numpy.ndarray` instance of the same shape and same data type. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + An instance of `numpy.ndarray` populated with content of `usm_ary`. + """ + return _copy_to_numpy(usm_ary) + + class Dummy: def __init__(self, iface): self.__sycl_usm_array_interface__ = iface @@ -138,9 +188,9 @@ def copy_same_dtype(dst, src): raise ValueError # check that memory regions do not overlap - if has_memory_overlap(dst, src): - tmp = copy_to_numpy(src) - copy_from_numpy_into(dst, tmp) + if _has_memory_overlap(dst, src): + tmp = _copy_to_numpy(src) + _copy_from_numpy_into(dst, tmp) return if (dst.flags & 1) and (src.flags & 1): @@ -184,10 +234,10 @@ def copy_same_shape(dst, src): return # check that memory regions do not overlap - if has_memory_overlap(dst, src): - tmp = copy_to_numpy(src) + if _has_memory_overlap(dst, src): + tmp = _copy_to_numpy(src) tmp = tmp.astype(dst.dtype) - copy_from_numpy_into(dst, tmp) + _copy_from_numpy_into(dst, tmp) return # simplify strides @@ -218,7 +268,7 @@ def copy_same_shape(dst, src): mdst.copy_from_host(tmp.view("u1")) -def copy_from_usm_ndarray_to_usm_ndarray(dst, src): +def _copy_from_usm_ndarray_to_usm_ndarray(dst, src): if type(dst) is not dpt.usm_ndarray or type(src) is not dpt.usm_ndarray: raise TypeError @@ -389,7 +439,7 @@ def astype(usm_ary, newdtype, order="K", casting="unsafe", copy=True): buffer=R.usm_data, strides=new_strides, ) - copy_from_usm_ndarray_to_usm_ndarray(R, usm_ary) + _copy_from_usm_ndarray_to_usm_ndarray(R, usm_ary) return R else: return usm_ary diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index d0cce717b0..1499146e13 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -20,6 +20,7 @@ import dpctl.memory as dpm import dpctl.tensor as dpt import dpctl.utils +from dpctl.tensor._device import normalize_queue_device _empty_tuple = tuple() _host_set = frozenset([None]) @@ -72,29 +73,6 @@ def _array_info_sequence(li): return (n,) + dim, dt, device -def _normalize_queue_device(q=None, d=None): - if q is None: - d = dpt._device.Device.create_device(d) - return d.sycl_queue - else: - if not isinstance(q, dpctl.SyclQueue): - raise TypeError(f"Expected dpctl.SyclQueue, got {type(q)}") - if d is None: - return q - d = dpt._device.Device.create_device(d) - qq = dpctl.utils.get_execution_queue( - ( - q, - d.sycl_queue, - ) - ) - if qq is None: - raise TypeError( - "sycl_queue and device keywords can not be both specified" - ) - return qq - - def _asarray_from_usm_ndarray( usm_ndary, dtype=None, @@ -115,7 +93,7 @@ def _asarray_from_usm_ndarray( exec_q = dpctl.utils.get_execution_queue( [usm_ndary.sycl_queue, sycl_queue] ) - copy_q = _normalize_queue_device(q=sycl_queue, d=exec_q) + copy_q = normalize_queue_device(sycl_queue=sycl_queue, device=exec_q) else: copy_q = usm_ndary.sycl_queue # Conditions for zero copy: @@ -194,7 +172,7 @@ def _asarray_from_numpy_ndarray( usm_type = "device" if dtype is None: dtype = ary.dtype - copy_q = _normalize_queue_device(q=None, d=sycl_queue) + copy_q = normalize_queue_device(sycl_queue=None, device=sycl_queue) f_contig = ary.flags["F"] c_contig = ary.flags["C"] fc_contig = f_contig or c_contig @@ -327,7 +305,9 @@ def asarray( ) # 5. Normalize device/sycl_queue [keep it None if was None] if device is not None or sycl_queue is not None: - sycl_queue = _normalize_queue_device(q=sycl_queue, d=device) + sycl_queue = normalize_queue_device( + sycl_queue=sycl_queue, device=device + ) # handle instance(obj, usm_ndarray) if isinstance(obj, dpt.usm_ndarray): @@ -459,7 +439,7 @@ def empty( raise TypeError( f"Expected usm_type to be of type str, got {type(usm_type)}" ) - sycl_queue = _normalize_queue_device(q=sycl_queue, d=device) + sycl_queue = normalize_queue_device(sycl_queue=sycl_queue, device=device) res = dpt.usm_ndarray( sh, dtype=dtype, diff --git a/dpctl/tensor/_device.py b/dpctl/tensor/_device.py index 7959735b4e..94564238e8 100644 --- a/dpctl/tensor/_device.py +++ b/dpctl/tensor/_device.py @@ -105,3 +105,54 @@ def __repr__(self): except TypeError: # This is a sub-device return repr(self.sycl_queue) + + +def normalize_queue_device(sycl_queue=None, device=None): + """ + normalize_queue_device(sycl_queue=None, device=None) + + Utility to process exclusive keyword arguments 'device' + and 'sycl_queue' in functions of `dpctl.tensor`. + + Args: + sycl_queue(:class:`dpctl.SyclQueue`, optional): + explicitly indicates where USM allocation is done + and the population code (if any) is executed. + Value `None` is interpreted as get the SYCL queue + from `device` keyword, or use default queue. + Default: None + device (string, :class:`dpctl.SyclDevice`, :class:`dpctl.SyclQueue, + :class:`dpctl.tensor.Device`, optional): + array-API keyword indicating non-partitioned SYCL device + where array is allocated. + Returns + :class:`dpctl.SyclQueue` object implied by either of provided + keywords. If both are None, `dpctl.SyclQueue()` is returned. + If both are specified and imply the same queue, `sycl_queue` + is returned. + Raises: + TypeError: if argument is not of the expected type, or keywords + imply incompatible queues. + """ + q = sycl_queue + d = device + if q is None: + d = Device.create_device(d) + return d.sycl_queue + else: + if not isinstance(q, dpctl.SyclQueue): + raise TypeError(f"Expected dpctl.SyclQueue, got {type(q)}") + if d is None: + return q + d = Device.create_device(d) + qq = dpctl.utils.get_execution_queue( + ( + q, + d.sycl_queue, + ) + ) + if qq is None: + raise TypeError( + "sycl_queue and device keywords can not be both specified" + ) + return qq diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 75131fd916..0b9a02a29b 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -884,13 +884,13 @@ cdef class usm_ndarray: except (ValueError, IndexError) as e: raise e from ._copy_utils import ( - copy_from_numpy_into, - copy_from_usm_ndarray_to_usm_ndarray, + _copy_from_numpy_into, + _copy_from_usm_ndarray_to_usm_ndarray, ) if isinstance(val, usm_ndarray): - copy_from_usm_ndarray_to_usm_ndarray(Xv, val) + _copy_from_usm_ndarray_to_usm_ndarray(Xv, val) else: - copy_from_numpy_into(Xv, np.asarray(val)) + _copy_from_numpy_into(Xv, np.asarray(val)) def __sub__(first, other): "See comment in __add__" From 5fa93f416fa5c1bdb49f2dc7458009a530e5b205 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 3 Dec 2021 18:07:12 -0600 Subject: [PATCH 087/229] Used dpt.to_numpy/dpt.from_numpy in test files This replaced local definition which had been removed --- dpctl/tests/test_usm_ndarray_ctor.py | 53 ++++------------------------ 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index b4d2ed7872..af5d973e64 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -18,13 +18,12 @@ import numbers import numpy as np -import numpy.lib.stride_tricks as np_st import pytest import dpctl import dpctl.memory as dpm import dpctl.tensor as dpt -from dpctl.tensor._usmarray import Device +from dpctl.tensor import Device @pytest.mark.parametrize( @@ -125,6 +124,7 @@ def test_properties(dt): assert isinstance(X.nbytes, numbers.Integral) assert isinstance(X.ndim, numbers.Integral) assert isinstance(X._pointer, numbers.Integral) + assert isinstance(X.device, Device) @pytest.mark.parametrize("func", [bool, float, int, complex]) @@ -198,49 +198,10 @@ def test_basic_slice(ind): assert S.dtype == X.dtype -def _from_numpy(np_ary, device=None, usm_type="shared"): - if type(np_ary) is np.ndarray: - if np_ary.flags["FORC"]: - x = np_ary - else: - x = np.ascontiguous(np_ary) - R = dpt.usm_ndarray( - np_ary.shape, - dtype=np_ary.dtype, - buffer=usm_type, - buffer_ctor_kwargs={ - "queue": Device.create_device(device).sycl_queue - }, - ) - R.usm_data.copy_from_host(x.reshape((-1)).view("|u1")) - return R - else: - raise ValueError("Expected numpy.ndarray, got {}".format(type(np_ary))) - - -def _to_numpy(usm_ary): - if type(usm_ary) is dpt.usm_ndarray: - usm_buf = usm_ary.usm_data - s = usm_buf.nbytes - host_buf = usm_buf.copy_to_host().view(usm_ary.dtype) - usm_ary_itemsize = usm_ary.itemsize - R_offset = ( - usm_ary.__sycl_usm_array_interface__["offset"] * usm_ary_itemsize - ) - R = np.ndarray((s,), dtype="u1", buffer=host_buf) - R = R[R_offset:].view(usm_ary.dtype) - R_strides = (usm_ary_itemsize * si for si in usm_ary.strides) - return np_st.as_strided(R, shape=usm_ary.shape, strides=R_strides) - else: - raise ValueError( - "Expected dpctl.tensor.usm_ndarray, got {}".format(type(usm_ary)) - ) - - def test_slice_constructor_1d(): Xh = np.arange(37, dtype="i4") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type="device") + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type="device") for ind in [ slice(1, None, 2), slice(0, None, 3), @@ -252,14 +213,14 @@ def test_slice_constructor_1d(): slice(None, None, -13), ]: assert np.array_equal( - _to_numpy(Xusm[ind]), Xh[ind] + dpt.asnumpy(Xusm[ind]), Xh[ind] ), "Failed for {}".format(ind) def test_slice_constructor_3d(): Xh = np.empty((37, 24, 35), dtype="i4") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type="device") + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type="device") for ind in [ slice(1, None, 2), slice(0, None, 3), @@ -272,7 +233,7 @@ def test_slice_constructor_3d(): (slice(None, None, -2), Ellipsis, None, 15), ]: assert np.array_equal( - _to_numpy(Xusm[ind]), Xh[ind] + dpt.to_numpy(Xusm[ind]), Xh[ind] ), "Failed for {}".format(ind) @@ -280,7 +241,7 @@ def test_slice_constructor_3d(): def test_slice_suai(usm_type): Xh = np.arange(0, 10, dtype="u1") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type=usm_type) + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type=usm_type) for ind in [slice(2, 3, None), slice(5, 7, None), slice(3, 9, None)]: assert np.array_equal( dpm.as_usm_memory(Xusm[ind]).copy_to_host(), Xh[ind] From 9c7426f21e7db04393d3d2d2784674ed94c464b8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 3 Dec 2021 18:18:33 -0600 Subject: [PATCH 088/229] Fixes #698 When processing sequence of sequences, pass requested array data type to numpy converter to ensure that the expected ValueError is raised --- dpctl/tensor/_ctors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index d0cce717b0..aea80bdc4a 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -388,7 +388,7 @@ def asarray( _, dt, devs = _array_info_sequence(obj) if devs == _host_set: return _asarray_from_numpy_ndarray( - np.asarray(obj, dt, order=order), + np.asarray(obj, dtype=dtype, order=order), dtype=dtype, usm_type=usm_type, sycl_queue=sycl_queue, From 8374c9f5ce5854f209c3a725381f11d9a3cf91f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 3 Dec 2021 19:06:22 -0600 Subject: [PATCH 089/229] Attempt to improve coverage of astype --- dpctl/tests/test_usm_ndarray_ctor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index af5d973e64..e6cee4a3a9 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -747,13 +747,13 @@ def test_to_device(): def test_astype(): - X = dpt.usm_ndarray((5, 5), "i4") + X = dpt.empty((5, 5), dtype="i4") X[:] = np.full((5, 5), 7, dtype="i4") Y = dpt.astype(X, "c16", order="C") assert np.allclose(dpt.to_numpy(Y), np.full((5, 5), 7, dtype="c16")) - Y = dpt.astype(X, "f2", order="K") - assert np.allclose(dpt.to_numpy(Y), np.full((5, 5), 7, dtype="f2")) - Y = dpt.astype(X, "i4", order="K", copy=False) + Y = dpt.astype(X[::2, ::-1], "f2", order="K") + assert np.allclose(dpt.to_numpy(Y), np.full(Y.shape, 7, dtype="f2")) + Y = dpt.astype(X[::2, ::-1], "i4", order="K", copy=False) assert Y.usm_data is X.usm_data From 99aba3bd4a1f4d963c97a15815160085241ffbed Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Mon, 6 Dec 2021 21:09:36 +0300 Subject: [PATCH 090/229] [Refactoring] Add variables for skip decorators (#684) --- dpctl/tests/test_sycl_queue_manager.py | 52 +++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dpctl/tests/test_sycl_queue_manager.py b/dpctl/tests/test_sycl_queue_manager.py index f9e572789d..0904c6912b 100644 --- a/dpctl/tests/test_sycl_queue_manager.py +++ b/dpctl/tests/test_sycl_queue_manager.py @@ -24,28 +24,36 @@ import dpctl - -@pytest.mark.skipif( +skip_no_platform = pytest.mark.skipif( not has_sycl_platforms(), reason="No SYCL platforms available" ) +skip_no_gpu = pytest.mark.skipif( + not has_gpu(), reason="No OpenCL GPU queues available" +) +skip_no_cpu = pytest.mark.skipif( + not has_cpu(), reason="No OpenCL CPU queues available" +) + + +@skip_no_platform def test_is_in_device_context_outside_device_ctxt(): assert not dpctl.is_in_device_context() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_gpu def test_is_in_device_context_inside_device_ctxt_gpu(): with dpctl.device_context("opencl:gpu:0"): assert dpctl.is_in_device_context() -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_cpu def test_is_in_device_context_inside_device_ctxt_cpu(): with dpctl.device_context("opencl:cpu:0"): assert dpctl.is_in_device_context() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_is_in_device_context_inside_nested_device_ctxt(): with dpctl.device_context("opencl:cpu:0"): with dpctl.device_context("opencl:gpu:0"): @@ -54,7 +62,7 @@ def test_is_in_device_context_inside_nested_device_ctxt(): assert not dpctl.is_in_device_context() -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_cpu def test_is_in_device_context_inside_nested_device_ctxt_cpu(): cpu = dpctl.SyclDevice("cpu") n = cpu.max_compute_units @@ -75,17 +83,13 @@ def test_is_in_device_context_inside_nested_device_ctxt_cpu(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_get_current_device_type_outside_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_platform +@skip_no_gpu def test_get_current_device_type_inside_device_ctxt(): assert dpctl.get_current_device_type() is not None @@ -95,8 +99,8 @@ def test_get_current_device_type_inside_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_cpu +@skip_no_gpu def test_get_current_device_type_inside_nested_device_ctxt(): assert dpctl.get_current_device_type() is not None @@ -110,15 +114,13 @@ def test_get_current_device_type_inside_nested_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_num_current_queues_outside_with_clause(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_num_current_queues_inside_with_clause(): with dpctl.device_context("opencl:cpu:0"): assert 1 == dpctl.get_num_activated_queues() @@ -127,8 +129,8 @@ def test_num_current_queues_inside_with_clause(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_num_current_queues_inside_threads(): from threading import Thread @@ -145,9 +147,7 @@ def SessionThread(): Session2.start() -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_get_current_backend(): dpctl.get_current_backend() dpctl.get_current_device_type() From 2e989fcd636066c38fd79a50e5e9683ab8966a9b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 6 Dec 2021 16:31:02 -0600 Subject: [PATCH 091/229] Collection of fixes and improvements to dpctl docstrings (#710) (#711) * Edits to submodule docstrings. * Change docstring summary for SyclContext class. * Fixes to docstring issues in dpctl.tensor. * Some more fixes to dpctl.tensor docstring issues. * Make usage signature of usm_ndarray ctor one long line Co-authored-by: Oleksandr Pavlyk Co-authored-by: Diptorup Deb --- dpctl/_sycl_context.pyx | 5 +++-- dpctl/memory/__init__.py | 6 ++---- dpctl/program/__init__.py | 7 +++---- dpctl/tensor/__init__.py | 14 +++----------- dpctl/tensor/_copy_utils.py | 11 +++++------ dpctl/tensor/_ctors.py | 8 ++------ dpctl/tensor/_device.py | 2 -- dpctl/tensor/_usmarray.pyx | 9 +-------- dpctl/utils/__init__.py | 2 +- 9 files changed, 20 insertions(+), 44 deletions(-) diff --git a/dpctl/_sycl_context.pyx b/dpctl/_sycl_context.pyx index 0499a21c06..56e0094582 100644 --- a/dpctl/_sycl_context.pyx +++ b/dpctl/_sycl_context.pyx @@ -86,8 +86,9 @@ cdef class _SyclContext: cdef class SyclContext(_SyclContext): """ SyclContext(arg=None) - Python class representing ``cl::sycl::context``. There are multiple - ways to create a :class:`dpctl.SyclContext` object: + A Python wrapper for the `sycl context`_ C++ class. + + There are multiple ways to create a :class:`dpctl.SyclContext` object: - Invoking the constructor with no arguments creates a context using the default selector. diff --git a/dpctl/memory/__init__.py b/dpctl/memory/__init__.py index 7b4032f1d7..593f03696e 100644 --- a/dpctl/memory/__init__.py +++ b/dpctl/memory/__init__.py @@ -15,10 +15,8 @@ # limitations under the License. """ - **Data Parallel Control Memory** - - `dpctl.memory` provides Python objects for untyped USM memory - container of bytes for each kind of USM pointers: shared pointers, + **Data Parallel Control Memory** provides Python objects for untyped USM + memory container of bytes for each kind of USM pointers: shared pointers, device pointers and host pointers. Shared and host pointers are accessible from both host and a device, diff --git a/dpctl/program/__init__.py b/dpctl/program/__init__.py index 58a46bb2eb..1d72e90e72 100644 --- a/dpctl/program/__init__.py +++ b/dpctl/program/__init__.py @@ -15,10 +15,9 @@ # limitations under the License. """ - **Data Parallel Control Program** - - `dpctl.program` module provides a way to create a SYCL kernel from either a - source string or SPIR-V binary file. + **Data Parallel Control Program** provides a way to create a SYCL kernel + from either an OpenCL program represented as a string or a SPIR-V binary + file. """ from ._program import ( diff --git a/dpctl/tensor/__init__.py b/dpctl/tensor/__init__.py index 944c80fa47..7aee5ebad1 100644 --- a/dpctl/tensor/__init__.py +++ b/dpctl/tensor/__init__.py @@ -15,17 +15,9 @@ # limitations under the License. """ - **Data Parallel Tensor Collection** - - ``dpctl.tensor`` is an experimental collection of tensor implementations - that will implement future Python data API - (https://data-apis.github.io/array-api/latest/). - - Available tensor implementations: - - ``numpy_usm_shared``: - Provides a ``numpy.ndarray`` sub-class whose underlying memory buffer - is allocated with a USM shared memory allocator. + **Data Parallel Tensor Collection** is a collection of tensor + implementations that implement Python data API + (https://data-apis.github.io/array-api/latest/) standard. """ diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index a158912d59..13866fc973 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -310,12 +310,11 @@ def copy(usm_ary, order="K"): following NumPy's conventions. The `order` keywords can be one of the following: - "C": C-contiguous memory layout - "F": Fotrant-contiguous memory layout - "A": Fotrant-contiguous if the input array is - F-contiguous, and C-contiguous otherwise - "K": match the layout of `usm_ary` as closely - as possible. + - "C": C-contiguous memory layout + - "F": Fortran-contiguous memory layout + - "A": Fortran-contiguous if the input array is also Fortran-contiguous, + otherwise C-contiguous + - "K": match the layout of `usm_ary` as closely as possible. """ if not isinstance(usm_ary, dpt.usm_ndarray): diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index 1d3c512c54..f080537682 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -233,9 +233,7 @@ def asarray( sycl_queue=None, order="K", ): - """asarray(obj, dtype=None, copy=None, order="K", - device=None, usm_type=None, sycl_queue=None) - + """ Converts `obj` to :class:`dpctl.tensor.usm_ndarray`. Args: @@ -395,9 +393,7 @@ def asarray( def empty( sh, dtype="f8", order="C", device=None, usm_type="device", sycl_queue=None ): - """dpctl.tensor.empty(shape, dtype="f8", order="C", device=None, - usm_type="device", sycl_queue=None) - + """ Creates `usm_ndarray` from uninitialized USM allocation. Args: diff --git a/dpctl/tensor/_device.py b/dpctl/tensor/_device.py index 94564238e8..c8f8bdef1b 100644 --- a/dpctl/tensor/_device.py +++ b/dpctl/tensor/_device.py @@ -35,8 +35,6 @@ def __new__(cls, *args, **kwargs): @classmethod def create_device(cls, dev): """ - Device.create_device(device) - Creates instance of Device from argument. Args: diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 0b9a02a29b..2062cf9d67 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -100,14 +100,7 @@ cdef class InternalUSMArrayError(Exception): cdef class usm_ndarray: - """ - usm_ndarray( - shape, dtype="|f8", strides=None, buffer='device', - offset=0, order='C', - buffer_ctor_kwargs=dict(), - array_namespace=None - ) - + """ usm_ndarray(shape, dtype="|f8", strides=None, buffer="device", offset=0, order="C", buffer_ctor_kwargs=dict(), array_namespace=None) See :class:`dpctl.memory.MemoryUSMShared` for allowed keyword arguments. diff --git a/dpctl/utils/__init__.py b/dpctl/utils/__init__.py index d4003fa825..7c52406ac0 100644 --- a/dpctl/utils/__init__.py +++ b/dpctl/utils/__init__.py @@ -15,7 +15,7 @@ # limitations under the License. """ -Collection of utilities. +A collection of utility functions. """ from ._compute_follows_data import get_execution_queue From ebdb0b09c83ac5449757878a8428a98fcb3f19bc Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 May 2021 15:33:50 -0500 Subject: [PATCH 092/229] Add a module to generate Sphinx rst files for Python modules. This module exposes generate_class_rst(cls) which would return string containing content of rst file, that is currently manually generated and placed into docs/docfiles/dpctl_pyapi/ Eg.: ``` import dpctl import generate_rst as gr print(gr.generate_class_rst(dpctl.SyclContext)) print(gr.generate_class_rst(dpctl.SyclQueue)) print(gr.generate_class_rst(dpctl.memory.MemoryUSMShared) ``` --- docs/generate_rst.py | 156 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 docs/generate_rst.py diff --git a/docs/generate_rst.py b/docs/generate_rst.py new file mode 100644 index 0000000000..777a95f5f6 --- /dev/null +++ b/docs/generate_rst.py @@ -0,0 +1,156 @@ +import inspect +import io + +import dpctl + +wrapper_descriptor = type(dpctl.SyclDevice.__init__) + +# known property in Cython extension class +getset_descriptor = type(dpctl.SyclDevice.name) +# known method (defined using def in Cython extension class) +cython_method_type = type(dpctl.SyclDevice.get_filter_string) +# known builtin method (defined using cpdef in Cython extension class) +cython_builtin_function_or_method_type = type(dpctl.SyclQueue.mro) + + +def get_public_class_name(cls): + if not inspect.isclass(cls): + raise TypeError("Expecting class, got {}".format(type(cls))) + modl = cls.__module__ + if modl: + modl = ".".join( + [comp for comp in modl.split(".") if not comp.startswith("_")] + ) + if modl: + res = ".".join([modl, cls.__qualname__]) + else: + res = cls.__qualname__ + return res + + +def is_class_property(o): + return isinstance(o, property) or (type(o) == getset_descriptor) + + +def is_class_method(o): + return inspect.ismethod(o) or ( + type(o) in [cython_method_type, cython_builtin_function_or_method_type] + ) + + +def get_filtered_names(cls, selector_func): + return [ + _name + for _name, _obj in inspect.getmembers(cls, selector_func) + if not _name.startswith("__") + ] + + +def generate_class_rst(cls): + if not inspect.isclass(cls): + raise TypeError("Expecting class, got {}".format(type(cls))) + + cls_qualname = get_public_class_name(cls) + # cls_property_names = get_filtered_names(cls, is_class_property) + # cls_method_names = get_filtered_names(cls, is_class_method) + rst_header = cls_qualname.split(".")[-1] + rst_module = ".".join(cls_qualname.split(".")[:-1]) + rst_header = "".join([".. _", rst_header, "_api:"]) + empty_line = "" + + def write_line(o, s): + o.write(s) + o.write("\n") + + def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): + write_line(o, indent + ".. rubric:: " + rubric_display) + write_line(o, "") + write_line(o, indent + ".. autosummary:: " + cls_qualname) + write_line(o, indent + indent + ":" + rubric_tag + ":") + write_line(o, "") + + def write_underlined(o, s, c): + write_line(o, s) + write_line(o, c * len(s)) + + with io.StringIO() as output: + write_line(output, rst_header) + write_line(output, empty_line) + marquee = "#" * len(cls_qualname) + write_line(output, marquee) + write_line(output, cls_qualname) + write_line(output, marquee) + write_line(output, empty_line) + + write_line(output, ".. currentmodule:: " + rst_module) + write_line(output, empty_line) + + write_line(output, ".. autoclass:: " + cls_qualname) + write_line(output, empty_line) + + indent = " " + + attributes_header = "Attributes" + write_rubric( + output, indent, attributes_header + ":", "attributes", cls_qualname + ) + public_methods_header = "Public methods" + write_rubric( + output, indent, public_methods_header + ":", "methods", cls_qualname + ) + private_methods_header = "Private methods" + write_rubric( + output, + indent, + private_methods_header + ":", + "private_methods", + cls_qualname, + ) + + write_underlined(output, "Detail", "=") + write_line(output, empty_line) + + # Attributes + all_attributes = get_filtered_names(cls, is_class_property) + if all_attributes: + write_underlined(output, attributes_header, "-") + write_line(output, empty_line) + for n in all_attributes: + write_line( + output, + ".. autoattribute:: " + ".".join([cls_qualname, n]), + ) + write_line(output, empty_line) + + # Methods, separated into public/private + all_methods = get_filtered_names(cls, is_class_method) + all_public_methods = [] + all_private_methods = [] + for _name in all_methods: + if _name.startswith("_"): + all_public_methods.append(_name) + else: + all_private_methods.append(_name) + + if all_public_methods: + write_underlined(output, public_methods_header, "-") + write_line(output, empty_line) + for n in all_public_methods: + write_line( + output, + ".. autoattribute:: " + ".".join([cls_qualname, n]), + ) + write_line(output, empty_line) + + # Private methods + if all_private_methods: + write_underlined(output, private_methods_header, "-") + write_line(output, empty_line) + for n in all_private_methods: + write_line( + output, + ".. autoattribute:: " + ".".join([cls_qualname, n]), + ) + write_line(output, empty_line) + + return output.getvalue() From e7ce0182645e714c7bc322aadafb9d4da8dc94e0 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 30 Nov 2021 14:57:21 -0600 Subject: [PATCH 093/229] Improvements to the generate_rst module. - Add a helper to generate the API docs for all classes in a module. - Add a function to generate a summary page for each module. - Automatically generates rst files for module summaries, classes, functions, exceptions, enums. - Recursively generate rst files for all submodules. --- docs/generate_rst.py | 483 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 429 insertions(+), 54 deletions(-) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 777a95f5f6..5320b8243d 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -1,19 +1,90 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" The module provides helper functions to generate API documentation for + dpctl's Python classes. +""" + +import enum import inspect import io +import os +import sys +from importlib import import_module +from pkgutil import iter_modules import dpctl -wrapper_descriptor = type(dpctl.SyclDevice.__init__) + +class MissingDocumentationError(Exception): + """ + Indicates that an undocumented class was found. + """ + + pass + # known property in Cython extension class -getset_descriptor = type(dpctl.SyclDevice.name) +_getset_descriptor = type(dpctl.SyclDevice.name) # known method (defined using def in Cython extension class) -cython_method_type = type(dpctl.SyclDevice.get_filter_string) +_cython_method_type = type(dpctl.SyclDevice.get_filter_string) # known builtin method (defined using cpdef in Cython extension class) -cython_builtin_function_or_method_type = type(dpctl.SyclQueue.mro) +_cython_builtin_function_or_method_type = type(dpctl.SyclQueue.mro) + +# Dictionary mapping internal module names to a readable string. so that we +# can use the module name to logically group functions. +function_groups = { + "dpctl._sycl_device_factory": "Device Selection Functions", + "dpctl._device_selection": "Device Selection Functions", + "dpctl._sycl_queue_manager": "Queue Management Functions", +} + + +def _get_module(module): + + try: + return sys.modules[module] + except KeyError: + raise ValueError( + module + "is not a valid module name or it is not loaded" + ) + + +def _write_line(output, s): + output.write(s) + output.write("\n") + + +def _write_empty_line(output): + _write_line(output, "") + + +def _write_marquee(o, s): + marquee = "#" * len(s) + _write_line(o, marquee) + _write_line(o, s) + _write_line(o, marquee) + + +def _write_underlined(o, s, c): + _write_line(o, s) + _write_line(o, c * len(s)) -def get_public_class_name(cls): +def _get_public_class_name(cls): if not inspect.isclass(cls): raise TypeError("Expecting class, got {}".format(type(cls))) modl = cls.__module__ @@ -28,17 +99,18 @@ def get_public_class_name(cls): return res -def is_class_property(o): - return isinstance(o, property) or (type(o) == getset_descriptor) +def _is_class_property(o): + return isinstance(o, property) or (type(o) == _getset_descriptor) -def is_class_method(o): +def _is_class_method(o): return inspect.ismethod(o) or ( - type(o) in [cython_method_type, cython_builtin_function_or_method_type] + type(o) + in [_cython_method_type, _cython_builtin_function_or_method_type] ) -def get_filtered_names(cls, selector_func): +def _get_filtered_names(cls, selector_func): return [ _name for _name, _obj in inspect.getmembers(cls, selector_func) @@ -46,47 +118,74 @@ def get_filtered_names(cls, selector_func): ] +def _group_functions(mod): + """Bin module functions into a set of logical groups. + + Args: + mod (object): A module whose functions will be grouped into bins + based on the ``function_groups`` dictionary. + + Returns: + [dict]: A dictionary containing grouping of functions in the + module. + """ + groups = {} + for name, obj in inspect.getmembers(mod): + if inspect.isbuiltin(obj) or inspect.isfunction(obj): + if obj.__module__ and obj.__module__ in function_groups: + try: + flist = groups[function_groups[obj.__module__]] + flist.append(obj) + except KeyError: + groups[function_groups[obj.__module__]] = [ + obj, + ] + else: + try: + flist = groups["Other Functions"] + flist.append(obj) + except KeyError: + groups["Other Functions"] = [ + obj, + ] + return groups + + def generate_class_rst(cls): + """Generate a rst file with the API documentation for a class. + + Raises: + TypeError: When the input is not a Python class + + Returns: + [str]: A string with rst nodes that can be written out to a file. + """ if not inspect.isclass(cls): raise TypeError("Expecting class, got {}".format(type(cls))) - cls_qualname = get_public_class_name(cls) - # cls_property_names = get_filtered_names(cls, is_class_property) - # cls_method_names = get_filtered_names(cls, is_class_method) + cls_qualname = _get_public_class_name(cls) rst_header = cls_qualname.split(".")[-1] rst_module = ".".join(cls_qualname.split(".")[:-1]) rst_header = "".join([".. _", rst_header, "_api:"]) - empty_line = "" - - def write_line(o, s): - o.write(s) - o.write("\n") def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): - write_line(o, indent + ".. rubric:: " + rubric_display) - write_line(o, "") - write_line(o, indent + ".. autosummary:: " + cls_qualname) - write_line(o, indent + indent + ":" + rubric_tag + ":") - write_line(o, "") - - def write_underlined(o, s, c): - write_line(o, s) - write_line(o, c * len(s)) + _write_line(o, indent + ".. rubric:: " + rubric_display) + _write_empty_line(o) + _write_line(o, indent + ".. autoautosummary:: " + cls_qualname) + _write_line(o, indent + indent + ":" + rubric_tag + ":") + _write_empty_line(o) with io.StringIO() as output: - write_line(output, rst_header) - write_line(output, empty_line) - marquee = "#" * len(cls_qualname) - write_line(output, marquee) - write_line(output, cls_qualname) - write_line(output, marquee) - write_line(output, empty_line) + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, cls_qualname) + _write_empty_line(output) - write_line(output, ".. currentmodule:: " + rst_module) - write_line(output, empty_line) + _write_line(output, ".. currentmodule:: " + rst_module) + _write_empty_line(output) - write_line(output, ".. autoclass:: " + cls_qualname) - write_line(output, empty_line) + _write_line(output, ".. autoclass:: " + cls_qualname) + _write_empty_line(output) indent = " " @@ -107,23 +206,23 @@ def write_underlined(o, s, c): cls_qualname, ) - write_underlined(output, "Detail", "=") - write_line(output, empty_line) + _write_underlined(output, "Detail", "=") + _write_empty_line(output) # Attributes - all_attributes = get_filtered_names(cls, is_class_property) + all_attributes = _get_filtered_names(cls, _is_class_property) if all_attributes: - write_underlined(output, attributes_header, "-") - write_line(output, empty_line) + _write_underlined(output, attributes_header, "-") + _write_empty_line(output) for n in all_attributes: - write_line( + _write_line( output, ".. autoattribute:: " + ".".join([cls_qualname, n]), ) - write_line(output, empty_line) + _write_empty_line(output) # Methods, separated into public/private - all_methods = get_filtered_names(cls, is_class_method) + all_methods = _get_filtered_names(cls, _is_class_method) all_public_methods = [] all_private_methods = [] for _name in all_methods: @@ -133,24 +232,300 @@ def write_underlined(o, s, c): all_private_methods.append(_name) if all_public_methods: - write_underlined(output, public_methods_header, "-") - write_line(output, empty_line) + _write_underlined(output, public_methods_header, "-") + _write_empty_line(output) for n in all_public_methods: - write_line( + _write_line( output, ".. autoattribute:: " + ".".join([cls_qualname, n]), ) - write_line(output, empty_line) + _write_empty_line(output) # Private methods if all_private_methods: - write_underlined(output, private_methods_header, "-") - write_line(output, empty_line) + _write_underlined(output, private_methods_header, "-") + _write_empty_line(output) for n in all_private_methods: - write_line( + _write_line( output, ".. autoattribute:: " + ".".join([cls_qualname, n]), ) - write_line(output, empty_line) return output.getvalue() + + +def generate_module_summary_rst(module): + """[summary] + + Args: + module ([str]): [description] + + Returns: + [type]: [description] + """ + rst_header = "".join([".. _", module, "_pyapi:"]) + pagename = module + " API" + indent = " " + + def _write_table_header(o): + _write_line(o, ".. list-table::") + _write_line(o, indent + ":widths: 25,50") + _write_empty_line(o) + + def _write_submodules_summary_table(o, mod): + _write_table_header(o) + for submod in iter_modules(mod.__path__): + if submod.ispkg: + _write_line( + o, + indent + + "* - :ref:`" + + mod.__name__ + + "." + + submod.name + + "_api`", + ) + _submod = import_module( + module + "." + submod.name, mod.__name__ + ) + if not _submod.__doc__: + raise MissingDocumentationError( + mod.__name__ + + "." + + _submod.__doc__ + + " is not documented." + ) + mod_summary = _submod.__doc__.split("\n")[0] + _write_line(o, indent + " - " + mod_summary) + _write_empty_line(o) + + def _write_classes_summary_table(o, mod): + _write_table_header(o) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and not ( + issubclass(obj, enum.Enum) or issubclass(obj, Exception) + ): + _write_line(o, indent + "* - :ref:`" + name + "_api`") + # For classes, the first line of the docstring is the + # signature. So we skip that line to pick up the summary. + if not obj.__doc__: + raise MissingDocumentationError( + mod.__name__ + "." + obj.__doc__ + " is not documented." + ) + cls_summary = obj.__doc__.split("\n")[1] + _write_line(o, indent + " - " + cls_summary) + _write_empty_line(o) + + def _write_enums_summary_table(o, mod): + _write_table_header(o) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, enum.Enum): + # FIXME link into the page pointing to the actual doc + # section for the enum. + _write_line( + o, indent + "* - :ref:`" + mod.__name__ + "_enum_api`" + ) + if not obj.__doc__: + raise MissingDocumentationError( + mod.__name__ + "." + obj.__doc__ + " is not documented." + ) + enum_summary = obj.__doc__.split("\n")[0] + _write_line(o, indent + " - " + enum_summary) + _write_empty_line(o) + + def _write_exceptions_summary_table(o, mod): + _write_table_header(o) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, Exception): + # FIXME link into the page pointing to the actual doc + # section for the exception. + _write_line( + o, indent + "* - :ref:`" + mod.__name__ + "_exception_api`" + ) + if not obj.__doc__: + raise MissingDocumentationError( + mod.__name__ + "." + obj.__doc__ + " is not documented." + ) + # For classes, the first line of the docstring is the + # signature. So we skip that line to pick up the summary. + excp_summary = obj.__doc__.split("\n")[1] + _write_line(o, indent + " - " + excp_summary) + _write_empty_line(o) + + def _write_functions_summary_table(o, mod, fnobj_list): + _write_table_header(o) + for fnobj in fnobj_list: + # FIXME link into the page pointing to the actual doc + # section for the exception. + _write_line( + o, indent + "* - :ref:`" + mod.__name__ + "_functions_api`" + ) + if not fnobj.__doc__: + raise MissingDocumentationError( + mod.__name__ + "." + fnobj.__doc__ + " is not documented." + ) + # For functions, the first line of the docstring is the + # signature. So we skip that line to pick up the summary. + fn_summary = fnobj.__doc__.split("\n")[1] + _write_line(o, indent + " - " + fn_summary) + _write_empty_line(o) + + def _write_function_groups_summary(o, mod, groups): + for group in groups: + _write_empty_line(o) + _write_underlined(o, group, "-") + _write_empty_line(o) + _write_functions_summary_table(o, mod, groups[group]) + + mod = _get_module(module) + + if not mod.__doc__: + raise MissingDocumentationError(mod.__name__ + " is not documented.") + + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + _write_line(output, ".. automodule:: " + module) + _write_empty_line(output) + _write_underlined(output, "Sub-modules", "-") + _write_empty_line(output) + _write_submodules_summary_table(output, mod) + _write_underlined(output, "Classes", "-") + _write_empty_line(output) + _write_classes_summary_table(output, mod) + _write_underlined(output, "Enums", "-") + _write_empty_line(output) + _write_enums_summary_table(output, mod) + _write_underlined(output, "Exceptions", "-") + _write_empty_line(output) + _write_exceptions_summary_table(output, mod) + _write_function_groups_summary(output, mod, _group_functions(mod)) + + return output.getvalue() + + +def generate_rst_for_all_classes(module, outputpath): + """Generates rst API docs for all classes in a module and writes them to + given path. + + Args: + module ([str]): Name of module that needs to be documented + outputpath ([str]): Path where the rst files are to be saved. + """ + mod = _get_module(module) + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and not ( + issubclass(obj, enum.Enum) or issubclass(obj, Exception) + ): + out = outputpath + "/" + name + ".rst" + with open(out, "w") as rst_file: + rst_file.write(generate_class_rst(obj)) + + +def generate_rst_for_all_functions(module, outputpath): + mod = _get_module(module) + groups = _group_functions(mod) + + rst_header = "".join([".. _", module, "_functions_api:"]) + pagename = module + " Functions" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + def _write_function_autodocs(o, groups): + for group, fnlist in groups.items(): + _write_empty_line(o) + _write_underlined(o, group, "-") + _write_empty_line(o) + for fn in fnlist: + _write_line(output, ".. autofunction:: " + fn.__name__) + + out = outputpath + "/" + module + "_functions_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + _write_function_autodocs(output, groups) + rst_file.write(output.getvalue()) + + +def generate_rst_for_all_exceptions(module, outputpath): + mod = _get_module(module) + rst_header = "".join([".. _", module, "_exception_api:"]) + pagename = module + " Exceptions" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + out = outputpath + "/" + module + "_exception_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, Exception): + _write_line(output, ".. autoexception:: " + obj.__name__) + + rst_file.write(output.getvalue()) + + +def generate_rst_for_all_enums(module, outputpath): + mod = _get_module(module) + indent = " " + rst_header = "".join([".. _", module, "_enum_api:"]) + pagename = module + " Enums" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + out = outputpath + "/" + module + "_enum_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, enum.Enum): + _write_line(output, ".. autoclass:: " + obj.__name__) + _write_line(output, indent + ":members:") + + rst_file.write(output.getvalue()) + + +def generate_all(module, outputpath): + mod = _get_module(module) + out = outputpath + "/" + module + "_pyapi.rst" + # Generate a summary page for the module's API + with open(out, "w") as rst_file: + rst_file.write(generate_module_summary_rst(module)) + # Generate supporting pages for the module + generate_rst_for_all_classes(module, outputpath) + generate_rst_for_all_enums(module, outputpath) + generate_rst_for_all_exceptions(module, outputpath) + generate_rst_for_all_functions(module, outputpath) + + # Now recurse into any submodule and generate all for them too. + for submod in iter_modules(mod.__path__): + if submod.ispkg: + generate_all(module + "." + submod.name, outputpath) From 945f42c0b5c8c570470b3ec8a14de684503cd989 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 2 Dec 2021 12:34:11 -0600 Subject: [PATCH 094/229] Insert FIXME notes when doc summary is missing. - Adjusted generator to insert fix-me notes when docstring for a public object is not found. - Removed manually written rst files for Python classes - Modified index files to point to generated dpctl_pyapi --- docs/.gitignore | 1 + docs/CMakeLists.txt | 2 + docs/docfiles/dpctl.memory_api.rst | 30 ------- docs/docfiles/dpctl.program_api.rst | 29 ------- docs/docfiles/dpctl.tensor_api.rst | 9 --- docs/docfiles/dpctl_pyapi.rst | 92 ---------------------- docs/docfiles/dpctl_pyapi/SyclContext.rst | 45 ----------- docs/docfiles/dpctl_pyapi/SyclDevice.rst | 84 -------------------- docs/docfiles/dpctl_pyapi/SyclEvent.rst | 20 ----- docs/docfiles/dpctl_pyapi/SyclPlatform.rst | 35 -------- docs/docfiles/dpctl_pyapi/SyclQueue.rst | 53 ------------- docs/generate_rst.py | 55 ++++--------- docs/index_doxyrest.rst.in | 8 +- docs/index_no_doxyrest.rst.in | 6 +- 14 files changed, 25 insertions(+), 444 deletions(-) delete mode 100644 docs/docfiles/dpctl.memory_api.rst delete mode 100644 docs/docfiles/dpctl.program_api.rst delete mode 100644 docs/docfiles/dpctl.tensor_api.rst delete mode 100644 docs/docfiles/dpctl_pyapi.rst delete mode 100644 docs/docfiles/dpctl_pyapi/SyclContext.rst delete mode 100644 docs/docfiles/dpctl_pyapi/SyclDevice.rst delete mode 100644 docs/docfiles/dpctl_pyapi/SyclEvent.rst delete mode 100644 docs/docfiles/dpctl_pyapi/SyclPlatform.rst delete mode 100644 docs/docfiles/dpctl_pyapi/SyclQueue.rst diff --git a/docs/.gitignore b/docs/.gitignore index 27d532e775..c781c1805a 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,6 +1,7 @@ docs generated_docs docfiles/libsyclinterface +docfiles/dpctl api build conf.py diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 8e34d6b886..48d7cf905b 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -185,6 +185,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) find_package(Git) find_package(Sphinx REQUIRED) find_package(Doxygen REQUIRED) + if (DPCTL_ENABLE_DOXYREST) find_package(Lua REQUIRED) find_package(Doxyrest REQUIRED) @@ -207,4 +208,5 @@ _setup_doxygen() if(DPCTL_ENABLE_DOXYREST) _setup_doxyrest() endif() +_generate_rst() _setup_sphinx() diff --git a/docs/docfiles/dpctl.memory_api.rst b/docs/docfiles/dpctl.memory_api.rst deleted file mode 100644 index c70041cdba..0000000000 --- a/docs/docfiles/dpctl.memory_api.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _dpctl.memory_api: - -############ -dpctl.memory -############ - -.. automodule:: dpctl.memory - -Classes -------- - -.. autoclass:: dpctl.memory.MemoryUSMDevice - :members: - :inherited-members: - :undoc-members: - -.. autoclass:: dpctl.memory.MemoryUSMHost - :members: - :inherited-members: - :undoc-members: - -.. autoclass:: dpctl.memory.MemoryUSMShared - :members: - :inherited-members: - :undoc-members: - -Functions ---------- - -.. autofunction:: dpctl.memory.as_usm_memory diff --git a/docs/docfiles/dpctl.program_api.rst b/docs/docfiles/dpctl.program_api.rst deleted file mode 100644 index 64db9c55e4..0000000000 --- a/docs/docfiles/dpctl.program_api.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _dpctl.program_api: - -############# -dpctl.program -############# - -.. automodule:: dpctl.program - -Classes -------- - -.. autoclass:: dpctl.program.SyclKernel - :members: - :undoc-members: - -.. autoclass:: dpctl.program.SyclProgram - :members: - :undoc-members: - -Exceptions ----------- - -.. autoexception:: dpctl.program.SyclProgramCompilationError - -Functions ---------- - -.. autofunction:: create_program_from_source -.. autofunction:: create_program_from_spirv diff --git a/docs/docfiles/dpctl.tensor_api.rst b/docs/docfiles/dpctl.tensor_api.rst deleted file mode 100644 index 837aa36ffb..0000000000 --- a/docs/docfiles/dpctl.tensor_api.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _dpctl.tensor_api: - -############ -dpctl.tensor -############ - -.. automodule:: dpctl.tensor - :members: - :undoc-members: diff --git a/docs/docfiles/dpctl_pyapi.rst b/docs/docfiles/dpctl_pyapi.rst deleted file mode 100644 index c7c66c63d8..0000000000 --- a/docs/docfiles/dpctl_pyapi.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. _dpctl_pyapi: - -################ -dpctl Python API -################ - -.. currentmodule:: dpctl - -.. automodule:: dpctl - -Sub-modules ------------ - - :mod:`dpctl.memory` - USM allocators and deallocators and classes that implement Python's - `buffer protocol`_. - :mod:`dpctl.tensor` - Implementation of different types of tensor classes that use USM memory. - :mod:`dpctl.program` - Experimental wrappers for SYCL 1.2 `Program` - and `Kernel` classes. - The module is going to be refactored in the future to support SYCL - 2020's ``sycl::kernel_bundle`` feature and the wrapper for the ``sycl::program`` - class is going to be removed. - -Classes -------- - -.. list-table:: - :widths: 25,50 - - * - :ref:`SyclPlatform_api` - - A Python class representing a `SYCL platform `_. - * - :ref:`SyclDevice_api` - - A Python class representing a `SYCL device `_. - * - :ref:`SyclQueue_api` - - A Python class representing a `SYCL queue `_. - * - :ref:`SyclContext_api` - - A Python class representing a `SYCL context `_. - * - :ref:`SyclEvent_api` - - A Python class representing a `SYCL event `_. - -Enumerations ------------- - -.. autoclass:: dpctl.backend_type - :members: - -.. autoclass:: dpctl.device_type - :members: - -Exceptions ----------- - -.. autoexception:: dpctl.SyclKernelInvalidRangeError -.. autoexception:: dpctl.SyclKernelSubmitError -.. autoexception:: dpctl.SyclQueueCreationError - -Device Selection Functions --------------------------- - -.. autofunction:: get_num_devices -.. autofunction:: get_devices -.. autofunction:: has_host_device -.. autofunction:: has_cpu_devices -.. autofunction:: has_gpu_devices -.. autofunction:: has_accelerator_devices -.. autofunction:: select_default_device -.. autofunction:: select_cpu_device -.. autofunction:: select_gpu_device -.. autofunction:: select_host_device -.. autofunction:: select_accelerator_device - -Queue Management Functions --------------------------- - -.. autofunction:: device_context -.. autofunction:: get_current_backend -.. autofunction:: get_current_device_type -.. autofunction:: get_current_queue -.. autofunction:: get_num_activated_queues -.. autofunction:: is_in_device_context -.. autofunction:: set_global_queue - -Other Helper Functions ----------------------- -.. autofunction:: get_platforms -.. autofunction:: lsplatform - -.. _Section 4.6: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes -.. _SYCL 2020 spec: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html -.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html diff --git a/docs/docfiles/dpctl_pyapi/SyclContext.rst b/docs/docfiles/dpctl_pyapi/SyclContext.rst deleted file mode 100644 index 8ec8728d04..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclContext.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. _SyclContext_api: - -################# -dpctl.SyclContext -################# - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclContext - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclContext - :attributes: - - .. rubric:: Private methods: - - .. autoautosummary:: dpctl.SyclContext - :private_methods: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclContext - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclContext.device_count - - -Private methods ---------------- - -.. autofunction:: dpctl.SyclContext._get_capsule - - -Public methods --------------- - -.. autofunction:: dpctl.SyclContext.addressof_ref -.. autofunction:: dpctl.SyclContext.get_devices diff --git a/docs/docfiles/dpctl_pyapi/SyclDevice.rst b/docs/docfiles/dpctl_pyapi/SyclDevice.rst deleted file mode 100644 index 43a13ce2fe..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclDevice.rst +++ /dev/null @@ -1,84 +0,0 @@ -.. _SyclDevice_api: - -################ -dpctl.SyclDevice -################ - -.. currentmodule:: dpctl - -.. autoclass:: SyclDevice - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclDevice - :attributes: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclDevice - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclDevice.backend -.. autoattribute:: dpctl.SyclDevice.default_selector_score -.. autoattribute:: dpctl.SyclDevice.device_type -.. autoattribute:: dpctl.SyclDevice.driver_version -.. autoattribute:: dpctl.SyclDevice.filter_string -.. autoattribute:: dpctl.SyclDevice.has_aspect_accelerator -.. autoattribute:: dpctl.SyclDevice.has_aspect_cpu -.. autoattribute:: dpctl.SyclDevice.has_aspect_custom -.. autoattribute:: dpctl.SyclDevice.has_aspect_fp16 -.. autoattribute:: dpctl.SyclDevice.has_aspect_fp64 -.. autoattribute:: dpctl.SyclDevice.has_aspect_gpu -.. autoattribute:: dpctl.SyclDevice.has_aspect_host -.. autoattribute:: dpctl.SyclDevice.has_aspect_image -.. autoattribute:: dpctl.SyclDevice.has_aspect_int64_base_atomics -.. autoattribute:: dpctl.SyclDevice.has_aspect_int64_extended_atomics -.. autoattribute:: dpctl.SyclDevice.has_aspect_online_compiler -.. autoattribute:: dpctl.SyclDevice.has_aspect_online_linker -.. autoattribute:: dpctl.SyclDevice.has_aspect_queue_profiling -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_device_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_host_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_restricted_shared_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_shared_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_system_allocator -.. autoattribute:: dpctl.SyclDevice.image_2d_max_height -.. autoattribute:: dpctl.SyclDevice.image_2d_max_width -.. autoattribute:: dpctl.SyclDevice.image_3d_max_depth -.. autoattribute:: dpctl.SyclDevice.image_3d_max_height -.. autoattribute:: dpctl.SyclDevice.image_3d_max_width -.. autoattribute:: dpctl.SyclDevice.is_accelerator -.. autoattribute:: dpctl.SyclDevice.is_cpu -.. autoattribute:: dpctl.SyclDevice.is_gpu -.. autoattribute:: dpctl.SyclDevice.is_host -.. autoattribute:: dpctl.SyclDevice.max_compute_units -.. autoattribute:: dpctl.SyclDevice.max_num_sub_groups -.. autoattribute:: dpctl.SyclDevice.max_read_image_args -.. autoattribute:: dpctl.SyclDevice.max_work_group_size -.. autoattribute:: dpctl.SyclDevice.max_work_item_dims -.. autoattribute:: dpctl.SyclDevice.max_work_item_sizes -.. autoattribute:: dpctl.SyclDevice.max_write_image_args -.. autoattribute:: dpctl.SyclDevice.name -.. autoattribute:: dpctl.SyclDevice.parent_device -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_char -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_double -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_float -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_half -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_int -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_long -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_short -.. autoattribute:: dpctl.SyclDevice.sub_group_independent_forward_progress -.. autoattribute:: dpctl.SyclDevice.vendor - -Public methods --------------- - -.. autofunction:: dpctl.SyclDevice.addressof_ref -.. autofunction:: dpctl.SyclDevice.create_sub_devices -.. autofunction:: dpctl.SyclDevice.get_filter_string -.. autofunction:: dpctl.SyclDevice.print_device_info diff --git a/docs/docfiles/dpctl_pyapi/SyclEvent.rst b/docs/docfiles/dpctl_pyapi/SyclEvent.rst deleted file mode 100644 index bf6dbdef62..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclEvent.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _SyclEvent_api: - -############### -dpctl.SyclEvent -############### - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclEvent - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclEvent - :methods: - -Detail -====== - -.. autofunction:: dpctl.SyclEvent.addressof_ref -.. autofunction:: dpctl.SyclEvent.wait diff --git a/docs/docfiles/dpctl_pyapi/SyclPlatform.rst b/docs/docfiles/dpctl_pyapi/SyclPlatform.rst deleted file mode 100644 index fa5dafe93d..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclPlatform.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. _SyclPlatform_api: - -################## -dpctl.SyclPlatform -################## - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclPlatform - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclPlatform - :attributes: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclPlatform - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclPlatform.backend -.. autoattribute:: dpctl.SyclPlatform.name -.. autoattribute:: dpctl.SyclPlatform.vendor -.. autoattribute:: dpctl.SyclPlatform.version - -Public methods --------------- - -.. autofunction:: dpctl.SyclPlatform.print_platform_info diff --git a/docs/docfiles/dpctl_pyapi/SyclQueue.rst b/docs/docfiles/dpctl_pyapi/SyclQueue.rst deleted file mode 100644 index a48018d6e1..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclQueue.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _SyclQueue_api: - -############### -dpctl.SyclQueue -############### - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclQueue - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclQueue - :attributes: - - .. rubric:: Private methods: - - .. autoautosummary:: dpctl.SyclQueue - :private_methods: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclQueue - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclQueue.is_in_order -.. autoattribute:: dpctl.SyclQueue.sycl_context -.. autoattribute:: dpctl.SyclQueue.sycl_device - -Private methods ---------------- - -.. autofunction:: dpctl.SyclQueue._get_capsule - - -Public methods --------------- - -.. autofunction:: dpctl.SyclQueue.addressof_ref -.. autofunction:: dpctl.SyclQueue.get_sycl_backend -.. autofunction:: dpctl.SyclQueue.get_sycl_context -.. autofunction:: dpctl.SyclQueue.get_sycl_device -.. autofunction:: dpctl.SyclQueue.mem_advise -.. autofunction:: dpctl.SyclQueue.memcpy -.. autofunction:: dpctl.SyclQueue.prefetch -.. autofunction:: dpctl.SyclQueue.submit -.. autofunction:: dpctl.SyclQueue.wait diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 5320b8243d..7f75b8cb56 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -28,15 +28,6 @@ import dpctl - -class MissingDocumentationError(Exception): - """ - Indicates that an undocumented class was found. - """ - - pass - - # known property in Cython extension class _getset_descriptor = type(dpctl.SyclDevice.name) # known method (defined using def in Cython extension class) @@ -267,6 +258,16 @@ def generate_module_summary_rst(module): pagename = module + " API" indent = " " + def _safe_get_docs(obj, i=0): + docstr = getattr(obj, "__doc__") + if not isinstance(docstr, str): + docstr = f"[FIXME]: {type(obj)} does not have a docstring" + return docstr + docstr = docstr.split("\n") + if len(docstr) < i + 1: + return f"[FIXME]: {type(obj)} has a docstring with no summary" + return docstr[i] + def _write_table_header(o): _write_line(o, ".. list-table::") _write_line(o, indent + ":widths: 25,50") @@ -288,14 +289,7 @@ def _write_submodules_summary_table(o, mod): _submod = import_module( module + "." + submod.name, mod.__name__ ) - if not _submod.__doc__: - raise MissingDocumentationError( - mod.__name__ - + "." - + _submod.__doc__ - + " is not documented." - ) - mod_summary = _submod.__doc__.split("\n")[0] + mod_summary = _safe_get_docs(_submod) _write_line(o, indent + " - " + mod_summary) _write_empty_line(o) @@ -308,11 +302,7 @@ def _write_classes_summary_table(o, mod): _write_line(o, indent + "* - :ref:`" + name + "_api`") # For classes, the first line of the docstring is the # signature. So we skip that line to pick up the summary. - if not obj.__doc__: - raise MissingDocumentationError( - mod.__name__ + "." + obj.__doc__ + " is not documented." - ) - cls_summary = obj.__doc__.split("\n")[1] + cls_summary = _safe_get_docs(obj, 1) _write_line(o, indent + " - " + cls_summary) _write_empty_line(o) @@ -325,11 +315,7 @@ def _write_enums_summary_table(o, mod): _write_line( o, indent + "* - :ref:`" + mod.__name__ + "_enum_api`" ) - if not obj.__doc__: - raise MissingDocumentationError( - mod.__name__ + "." + obj.__doc__ + " is not documented." - ) - enum_summary = obj.__doc__.split("\n")[0] + enum_summary = _safe_get_docs(obj) _write_line(o, indent + " - " + enum_summary) _write_empty_line(o) @@ -342,13 +328,9 @@ def _write_exceptions_summary_table(o, mod): _write_line( o, indent + "* - :ref:`" + mod.__name__ + "_exception_api`" ) - if not obj.__doc__: - raise MissingDocumentationError( - mod.__name__ + "." + obj.__doc__ + " is not documented." - ) # For classes, the first line of the docstring is the # signature. So we skip that line to pick up the summary. - excp_summary = obj.__doc__.split("\n")[1] + excp_summary = _safe_get_docs(obj, 1) _write_line(o, indent + " - " + excp_summary) _write_empty_line(o) @@ -360,13 +342,9 @@ def _write_functions_summary_table(o, mod, fnobj_list): _write_line( o, indent + "* - :ref:`" + mod.__name__ + "_functions_api`" ) - if not fnobj.__doc__: - raise MissingDocumentationError( - mod.__name__ + "." + fnobj.__doc__ + " is not documented." - ) # For functions, the first line of the docstring is the # signature. So we skip that line to pick up the summary. - fn_summary = fnobj.__doc__.split("\n")[1] + fn_summary = _safe_get_docs(fnobj, 1) _write_line(o, indent + " - " + fn_summary) _write_empty_line(o) @@ -379,9 +357,6 @@ def _write_function_groups_summary(o, mod, groups): mod = _get_module(module) - if not mod.__doc__: - raise MissingDocumentationError(mod.__name__ + " is not documented.") - with io.StringIO() as output: _write_line(output, rst_header) _write_empty_line(output) diff --git a/docs/index_doxyrest.rst.in b/docs/index_doxyrest.rst.in index 51e55562f9..cc3c17700b 100644 --- a/docs/index_doxyrest.rst.in +++ b/docs/index_doxyrest.rst.in @@ -7,10 +7,10 @@ docfiles/user_guides/QuickStart .. toctree:: - :maxdepth: 1 - :caption: API Documentation + :maxdepth: 1 + :caption: API Documentation - docfiles/dpctl_pyapi - @DOXYREST_OUTPUT_DIR_NAME@/index + docfiles/dpctl/dpctl_pyapi + @DOXYREST_OUTPUT_DIR_NAME@/index .. include:: ./docfiles/boilerplate.rst diff --git a/docs/index_no_doxyrest.rst.in b/docs/index_no_doxyrest.rst.in index fffbf63cb4..1f367ce2d8 100644 --- a/docs/index_no_doxyrest.rst.in +++ b/docs/index_no_doxyrest.rst.in @@ -7,9 +7,9 @@ docfiles/user_guides/QuickStart .. toctree:: - :maxdepth: 1 - :caption: API Documentation + :maxdepth: 1 + :caption: API Documentation - docfiles/dpctl_pyapi + docfiles/dpctl/dpctl_pyapi .. include:: ./docfiles/boilerplate.rst From 65df023471ec55dfd8e18b3bc341baf3feb867c9 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Thu, 2 Dec 2021 13:53:55 -0600 Subject: [PATCH 095/229] Fixes to generate_rst module - Fix links for functions and submodules on every module summary page. - Fix links for Enums and Exceptions summary tables on any module summary page. - Fix the swapped public and private method headers in class summary. - Function summary is now printed before enums and exceptions. - Summary tables are printed only when an object of that type exists. - The generate_rst module now supports argparse. - Improve how we search the summary line from the docstring. --- docs/generate_rst.py | 294 ++++++++++++++++++++++++++++++++----------- 1 file changed, 220 insertions(+), 74 deletions(-) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 7f75b8cb56..9fce0df955 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -15,9 +15,10 @@ # limitations under the License. """ The module provides helper functions to generate API documentation for - dpctl's Python classes. + dpctl and its members. """ +import argparse import enum import inspect import io @@ -45,7 +46,19 @@ def _get_module(module): + """Get the Python object for a module from a string providing the module's + name. + Args: + module ([str]): The name of a module to be searched in ``sys.modules``. + + Raises: + ValueError: If no corresponding module object was found for the string + module name. + + Returns: + [object]: A Python object representing a module. + """ try: return sys.modules[module] except KeyError: @@ -55,15 +68,32 @@ def _get_module(module): def _write_line(output, s): + """Write a line to specified out stream. + + Args: + output (``io.StringIO``): The string stream to be written. + s (str): The string that is to be written out as a line. + """ output.write(s) output.write("\n") def _write_empty_line(output): + """[summary] + + Args: + output ([type]): [description] + """ _write_line(output, "") def _write_marquee(o, s): + """[summary] + + Args: + o ([type]): [description] + s ([type]): [description] + """ marquee = "#" * len(s) _write_line(o, marquee) _write_line(o, s) @@ -71,11 +101,26 @@ def _write_marquee(o, s): def _write_underlined(o, s, c): + """[summary] + + Args: + o ([type]): [description] + s ([type]): [description] + c ([type]): [description] + """ _write_line(o, s) _write_line(o, c * len(s)) def _get_public_class_name(cls): + """[summary] + + Raises: + TypeError: [description] + + Returns: + [type]: [description] + """ if not inspect.isclass(cls): raise TypeError("Expecting class, got {}".format(type(cls))) modl = cls.__module__ @@ -91,10 +136,26 @@ def _get_public_class_name(cls): def _is_class_property(o): + """[summary] + + Args: + o ([type]): [description] + + Returns: + [type]: [description] + """ return isinstance(o, property) or (type(o) == _getset_descriptor) def _is_class_method(o): + """[summary] + + Args: + o ([type]): [description] + + Returns: + [type]: [description] + """ return inspect.ismethod(o) or ( type(o) in [_cython_method_type, _cython_builtin_function_or_method_type] @@ -102,6 +163,14 @@ def _is_class_method(o): def _get_filtered_names(cls, selector_func): + """[summary] + + Args: + selector_func ([type]): [description] + + Returns: + [type]: [description] + """ return [ _name for _name, _obj in inspect.getmembers(cls, selector_func) @@ -142,7 +211,7 @@ def _group_functions(mod): return groups -def generate_class_rst(cls): +def _generate_class_rst(cls): """Generate a rst file with the API documentation for a class. Raises: @@ -218,9 +287,9 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): all_private_methods = [] for _name in all_methods: if _name.startswith("_"): - all_public_methods.append(_name) - else: all_private_methods.append(_name) + else: + all_public_methods.append(_name) if all_public_methods: _write_underlined(output, public_methods_header, "-") @@ -245,7 +314,7 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): return output.getvalue() -def generate_module_summary_rst(module): +def _generate_module_summary_rst(module): """[summary] Args: @@ -255,18 +324,47 @@ def generate_module_summary_rst(module): [type]: [description] """ rst_header = "".join([".. _", module, "_pyapi:"]) - pagename = module + " API" + pagename = module indent = " " - def _safe_get_docs(obj, i=0): + def _get_doc_summary(obj): docstr = getattr(obj, "__doc__") if not isinstance(docstr, str): docstr = f"[FIXME]: {type(obj)} does not have a docstring" return docstr - docstr = docstr.split("\n") - if len(docstr) < i + 1: + # Let us stip out any newlines, tabs, etc. at the start of the docstr + docstr = docstr.lstrip() + # Check if a signature line is provided. The check only looks for + # something like "SyclContext(" + st = ( + len(obj.__name__ + "(") + if docstr.startswith(obj.__name__ + "(", 0) + else 0 + ) + # If an apparent signature line was seen, then locate the end of the + # signature line. + if st: + nOpens = 1 + for i, c in enumerate(docstr[st:]): + if c == "(": + nOpens += 1 + elif c == ")": + stop = i + nOpens -= 1 + if nOpens == 0: + break + st += stop + 1 + # Strip out the signature in the docstring. + docstr = docstr[st:] + # The hope is to find the first line (summary) from the docstring + # by searching for a period followed by a new line. Not foolproof, but + # a best-effort check. + docstr = " ".join( + docstr[0 : docstr.find(".\n") + 1].replace("\n", " ").split() + ) + if len(docstr) < 1: return f"[FIXME]: {type(obj)} has a docstring with no summary" - return docstr[i] + return docstr def _write_table_header(o): _write_line(o, ".. list-table::") @@ -274,77 +372,89 @@ def _write_table_header(o): _write_empty_line(o) def _write_submodules_summary_table(o, mod): - _write_table_header(o) - for submod in iter_modules(mod.__path__): - if submod.ispkg: + submods = [ + submod.name for submod in iter_modules(mod.__path__) if submod.ispkg + ] + if submods: + _write_empty_line(output) + _write_underlined(output, "Sub-modules", "-") + _write_empty_line(output) + _write_table_header(o) + for submod in submods: _write_line( o, indent + "* - :ref:`" + mod.__name__ + "." - + submod.name - + "_api`", + + submod + + "_pyapi`", ) - _submod = import_module( - module + "." + submod.name, mod.__name__ - ) - mod_summary = _safe_get_docs(_submod) + _submod = import_module(module + "." + submod, mod.__name__) + mod_summary = _get_doc_summary(_submod) _write_line(o, indent + " - " + mod_summary) - _write_empty_line(o) + _write_empty_line(o) def _write_classes_summary_table(o, mod): - _write_table_header(o) - for name, obj in inspect.getmembers(mod): - if inspect.isclass(obj) and not ( - issubclass(obj, enum.Enum) or issubclass(obj, Exception) + classes = [] + for mem_tup in inspect.getmembers(mod): + cls = mem_tup[1] + if inspect.isclass(cls) and not ( + issubclass(cls, enum.Enum) or issubclass(cls, Exception) ): - _write_line(o, indent + "* - :ref:`" + name + "_api`") + classes.append(cls) + if classes: + _write_underlined(o, "Classes", "-") + _write_empty_line(o) + _write_table_header(o) + for cls in classes: + _write_line(o, indent + "* - :class:`" + cls.__name__ + "`") # For classes, the first line of the docstring is the # signature. So we skip that line to pick up the summary. - cls_summary = _safe_get_docs(obj, 1) + cls_summary = _get_doc_summary(cls) _write_line(o, indent + " - " + cls_summary) - _write_empty_line(o) + _write_empty_line(o) def _write_enums_summary_table(o, mod): - _write_table_header(o) - for name, obj in inspect.getmembers(mod): - if inspect.isclass(obj) and issubclass(obj, enum.Enum): - # FIXME link into the page pointing to the actual doc - # section for the enum. - _write_line( - o, indent + "* - :ref:`" + mod.__name__ + "_enum_api`" - ) - enum_summary = _safe_get_docs(obj) + enums = [] + for mem_tup in inspect.getmembers(mod): + e = mem_tup[1] + if inspect.isclass(e) and (issubclass(e, enum.Enum)): + enums.append(e) + if enums: + _write_underlined(o, "Enums", "-") + _write_empty_line(o) + _write_table_header(o) + for e in enums: + _write_line(o, indent + "* - :class:`" + e.__name__ + "`") + enum_summary = _get_doc_summary(e) _write_line(o, indent + " - " + enum_summary) _write_empty_line(o) def _write_exceptions_summary_table(o, mod): - _write_table_header(o) - for name, obj in inspect.getmembers(mod): - if inspect.isclass(obj) and issubclass(obj, Exception): - # FIXME link into the page pointing to the actual doc - # section for the exception. - _write_line( - o, indent + "* - :ref:`" + mod.__name__ + "_exception_api`" - ) - # For classes, the first line of the docstring is the - # signature. So we skip that line to pick up the summary. - excp_summary = _safe_get_docs(obj, 1) - _write_line(o, indent + " - " + excp_summary) + exps = [] + for mem_tup in inspect.getmembers(mod): + e = mem_tup[1] + if inspect.isclass(e) and (issubclass(e, Exception)): + exps.append(e) + + if exps: + _write_underlined(o, "Exceptions", "-") + _write_empty_line(o) + _write_table_header(o) + for e in exps: + _write_line(o, indent + "* - :class:`" + e.__name__ + "`") + excep_summary = _get_doc_summary(e) + _write_line(o, indent + " - " + excep_summary) _write_empty_line(o) def _write_functions_summary_table(o, mod, fnobj_list): _write_table_header(o) for fnobj in fnobj_list: - # FIXME link into the page pointing to the actual doc - # section for the exception. - _write_line( - o, indent + "* - :ref:`" + mod.__name__ + "_functions_api`" - ) + _write_line(o, indent + "* - :func:`" + fnobj.__name__ + "()`") # For functions, the first line of the docstring is the # signature. So we skip that line to pick up the summary. - fn_summary = _safe_get_docs(fnobj, 1) + fn_summary = _get_doc_summary(fnobj) _write_line(o, indent + " - " + fn_summary) _write_empty_line(o) @@ -365,25 +475,16 @@ def _write_function_groups_summary(o, mod, groups): _write_line(output, ".. currentmodule:: " + module) _write_empty_line(output) _write_line(output, ".. automodule:: " + module) - _write_empty_line(output) - _write_underlined(output, "Sub-modules", "-") - _write_empty_line(output) _write_submodules_summary_table(output, mod) - _write_underlined(output, "Classes", "-") - _write_empty_line(output) _write_classes_summary_table(output, mod) - _write_underlined(output, "Enums", "-") - _write_empty_line(output) + _write_function_groups_summary(output, mod, _group_functions(mod)) _write_enums_summary_table(output, mod) - _write_underlined(output, "Exceptions", "-") - _write_empty_line(output) _write_exceptions_summary_table(output, mod) - _write_function_groups_summary(output, mod, _group_functions(mod)) return output.getvalue() -def generate_rst_for_all_classes(module, outputpath): +def _generate_rst_for_all_classes(module, outputpath): """Generates rst API docs for all classes in a module and writes them to given path. @@ -401,10 +502,19 @@ def generate_rst_for_all_classes(module, outputpath): ): out = outputpath + "/" + name + ".rst" with open(out, "w") as rst_file: - rst_file.write(generate_class_rst(obj)) + rst_file.write(_generate_class_rst(obj)) + +def _generate_rst_for_all_functions(module, outputpath): + """[summary] -def generate_rst_for_all_functions(module, outputpath): + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ mod = _get_module(module) groups = _group_functions(mod) @@ -436,7 +546,16 @@ def _write_function_autodocs(o, groups): rst_file.write(output.getvalue()) -def generate_rst_for_all_exceptions(module, outputpath): +def _generate_rst_for_all_exceptions(module, outputpath): + """[summary] + + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ mod = _get_module(module) rst_header = "".join([".. _", module, "_exception_api:"]) pagename = module + " Exceptions" @@ -461,7 +580,16 @@ def generate_rst_for_all_exceptions(module, outputpath): rst_file.write(output.getvalue()) -def generate_rst_for_all_enums(module, outputpath): +def _generate_rst_for_all_enums(module, outputpath): + """[summary] + + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ mod = _get_module(module) indent = " " rst_header = "".join([".. _", module, "_enum_api:"]) @@ -489,18 +617,36 @@ def generate_rst_for_all_enums(module, outputpath): def generate_all(module, outputpath): + """Recursively generate rst files for a root module and all its members. + + Args: + module ([str]): Name of a Python module + outputpath ([str]): Output directory + """ mod = _get_module(module) out = outputpath + "/" + module + "_pyapi.rst" # Generate a summary page for the module's API with open(out, "w") as rst_file: - rst_file.write(generate_module_summary_rst(module)) + rst_file.write(_generate_module_summary_rst(module)) # Generate supporting pages for the module - generate_rst_for_all_classes(module, outputpath) - generate_rst_for_all_enums(module, outputpath) - generate_rst_for_all_exceptions(module, outputpath) - generate_rst_for_all_functions(module, outputpath) + _generate_rst_for_all_classes(module, outputpath) + _generate_rst_for_all_enums(module, outputpath) + _generate_rst_for_all_exceptions(module, outputpath) + _generate_rst_for_all_functions(module, outputpath) # Now recurse into any submodule and generate all for them too. for submod in iter_modules(mod.__path__): if submod.ispkg: generate_all(module + "." + submod.name, outputpath) + + +parser = argparse.ArgumentParser("Generate rst files for Python source files") +parser.add_argument("--dir", help="Output directory", required=True) +parser.add_argument("--module", help="Python module", required=True) + +args = parser.parse_args() +outdir = args.dir +mod = args.module + +# Run generate_all +generate_all(mod, outdir) From 226f8dfdb3192a3b58046b189218a563f08058c1 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Sun, 5 Dec 2021 21:27:25 -0600 Subject: [PATCH 096/229] Add a static rst file with all urls. Include the file in generated rst files. --- docs/docfiles/urls.rst | 12 ++++++++++++ docs/generate_rst.py | 7 ++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 docs/docfiles/urls.rst diff --git a/docs/docfiles/urls.rst b/docs/docfiles/urls.rst new file mode 100644 index 0000000000..d78c1ba703 --- /dev/null +++ b/docs/docfiles/urls.rst @@ -0,0 +1,12 @@ +.. __urls: + +.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html +.. _Data API: https://data-apis.github.io/array-api/latest/ + +.. _Section 4.6: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes +.. _SYCL 2020 spec: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html +.. _sycl platform: https://sycl.readthedocs.io/en/latest/iface/platform.html +.. _sycl device: https://sycl.readthedocs.io/en/latest/iface/device.html +.. _sycl queue: https://sycl.readthedocs.io/en/latest/iface/queue.html +.. _sycl event: https://sycl.readthedocs.io/en/latest/iface/event.html +.. _sycl context: https://sycl.readthedocs.io/en/latest/iface/context.html diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 9fce0df955..e5baa46319 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -362,7 +362,7 @@ def _get_doc_summary(obj): docstr = " ".join( docstr[0 : docstr.find(".\n") + 1].replace("\n", " ").split() ) - if len(docstr) < 1: + if not docstr: return f"[FIXME]: {type(obj)} has a docstring with no summary" return docstr @@ -465,6 +465,10 @@ def _write_function_groups_summary(o, mod, groups): _write_empty_line(o) _write_functions_summary_table(o, mod, groups[group]) + def _write_include_urls(o): + _write_empty_line(o) + _write_line(o, ".. include:: ../urls.rst") + mod = _get_module(module) with io.StringIO() as output: @@ -480,6 +484,7 @@ def _write_function_groups_summary(o, mod, groups): _write_function_groups_summary(output, mod, _group_functions(mod)) _write_enums_summary_table(output, mod) _write_exceptions_summary_table(output, mod) + _write_include_urls(output) return output.getvalue() From 897da03b715a720d98b74814ba2a756ca9a5757a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 6 Dec 2021 08:48:57 -0600 Subject: [PATCH 097/229] Do not generate rubriks for classes of absent methods Case in point, `dpctl.SyclTimer` does not have neither public nor private methods, but rubric was there nonetheless. Now the rubric won't be generated of the the set of methods/attributes from the respective class is empty. --- docs/generate_rst.py | 69 +++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index e5baa46319..18ec206d1a 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -236,6 +236,18 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): _write_empty_line(o) with io.StringIO() as output: + # Attributes + all_attributes = _get_filtered_names(cls, _is_class_property) + # Methods, separated into public/private + all_methods = _get_filtered_names(cls, _is_class_method) + all_public_methods = [] + all_private_methods = [] + for _name in all_methods: + if _name.startswith("_"): + all_private_methods.append(_name) + else: + all_public_methods.append(_name) + _write_line(output, rst_header) _write_empty_line(output) _write_marquee(output, cls_qualname) @@ -248,29 +260,38 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): _write_empty_line(output) indent = " " - attributes_header = "Attributes" - write_rubric( - output, indent, attributes_header + ":", "attributes", cls_qualname - ) - public_methods_header = "Public methods" - write_rubric( - output, indent, public_methods_header + ":", "methods", cls_qualname - ) private_methods_header = "Private methods" - write_rubric( - output, - indent, - private_methods_header + ":", - "private_methods", - cls_qualname, - ) + public_methods_header = "Public methods" + + if all_attributes: + write_rubric( + output, + indent, + attributes_header + ":", + "attributes", + cls_qualname, + ) + if all_public_methods: + write_rubric( + output, + indent, + public_methods_header + ":", + "methods", + cls_qualname, + ) + if all_private_methods: + write_rubric( + output, + indent, + private_methods_header + ":", + "private_methods", + cls_qualname, + ) _write_underlined(output, "Detail", "=") _write_empty_line(output) - # Attributes - all_attributes = _get_filtered_names(cls, _is_class_property) if all_attributes: _write_underlined(output, attributes_header, "-") _write_empty_line(output) @@ -281,23 +302,13 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): ) _write_empty_line(output) - # Methods, separated into public/private - all_methods = _get_filtered_names(cls, _is_class_method) - all_public_methods = [] - all_private_methods = [] - for _name in all_methods: - if _name.startswith("_"): - all_private_methods.append(_name) - else: - all_public_methods.append(_name) - if all_public_methods: _write_underlined(output, public_methods_header, "-") _write_empty_line(output) for n in all_public_methods: _write_line( output, - ".. autoattribute:: " + ".".join([cls_qualname, n]), + ".. autofunction:: " + ".".join([cls_qualname, n]), ) _write_empty_line(output) @@ -308,7 +319,7 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): for n in all_private_methods: _write_line( output, - ".. autoattribute:: " + ".".join([cls_qualname, n]), + ".. autofunction:: " + ".".join([cls_qualname, n]), ) return output.getvalue() From 39a2a9e6cdb81b389ec3c628395935d4c9a1e5f8 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Mon, 6 Dec 2021 13:26:27 -0600 Subject: [PATCH 098/229] Add pages to table of contents on the summary pages. - Include sub-module rst pages and class rst pages to a hidden TOC on a module summary page. - Remove the "Details" heading line from class summary so that content gets added correctly to sidebar TOC. --- docs/generate_rst.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 18ec206d1a..05e1e1601d 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -112,6 +112,35 @@ def _write_underlined(o, s, c): _write_line(o, c * len(s)) +def _write_include_urls(o): + """[summary] + + Args: + o ([type]): [description] + """ + _write_empty_line(o) + _write_line(o, ".. include:: ../urls.rst") + + +def _write_hidden_toc(o, list_of_obj_names, prefix_str="", suffix_str=""): + """[summary] + + Args: + o ([type]): [description] + list_of_objs ([type]): [description] + prefix_str ([type]): [description] + suffix_str ([type]): [description] + """ + if not list_of_obj_names: + return + _write_line(o, ".. toctree::") + _write_line(o, " :hidden:") + _write_empty_line(o) + for obj in list_of_obj_names: + _write_line(o, " " + prefix_str + obj + suffix_str) + _write_empty_line(o) + + def _get_public_class_name(cls): """[summary] @@ -289,7 +318,6 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): cls_qualname, ) - _write_underlined(output, "Detail", "=") _write_empty_line(output) if all_attributes: @@ -322,6 +350,8 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): ".. autofunction:: " + ".".join([cls_qualname, n]), ) + _write_include_urls(output) + return output.getvalue() @@ -390,6 +420,7 @@ def _write_submodules_summary_table(o, mod): _write_empty_line(output) _write_underlined(output, "Sub-modules", "-") _write_empty_line(output) + _write_hidden_toc(output, submods, mod.__name__ + ".", "_pyapi") _write_table_header(o) for submod in submods: _write_line( @@ -408,15 +439,18 @@ def _write_submodules_summary_table(o, mod): def _write_classes_summary_table(o, mod): classes = [] + class_names = [] for mem_tup in inspect.getmembers(mod): cls = mem_tup[1] if inspect.isclass(cls) and not ( issubclass(cls, enum.Enum) or issubclass(cls, Exception) ): classes.append(cls) + class_names.append(mem_tup[0]) if classes: _write_underlined(o, "Classes", "-") _write_empty_line(o) + _write_hidden_toc(output, class_names) _write_table_header(o) for cls in classes: _write_line(o, indent + "* - :class:`" + cls.__name__ + "`") @@ -476,10 +510,6 @@ def _write_function_groups_summary(o, mod, groups): _write_empty_line(o) _write_functions_summary_table(o, mod, groups[group]) - def _write_include_urls(o): - _write_empty_line(o) - _write_line(o, ".. include:: ../urls.rst") - mod = _get_module(module) with io.StringIO() as output: From 8a7be62fac67f970d4e9b60aef3267442c0067fc Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 6 Dec 2021 16:41:53 -0600 Subject: [PATCH 099/229] Expanded specialized function grouping --- docs/generate_rst.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 05e1e1601d..c11c2b38fe 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -42,6 +42,13 @@ "dpctl._sycl_device_factory": "Device Selection Functions", "dpctl._device_selection": "Device Selection Functions", "dpctl._sycl_queue_manager": "Queue Management Functions", + "dpctl.tensor._ctors": "Array Construction", + "dpctl.tensor._copy_utils": "Array Construction", + "dpctl.tensor._dlpack": "Array Construction", + "dpctl.tensor._reshape": "Array Manipulation", + "dpctl.memory._memory": "Functions", + "dpctl.program._program": "Functions", + "dpctl.utils._compute_follows_data": "Functions", } From 3614d9009d36442f4f21b9811ab7e8b401fc521a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 2 Dec 2021 16:35:02 -0600 Subject: [PATCH 100/229] Use dir(obj) over obj.__dict__.keys() to get all attributes Previous code was only looking up dynamically defined attributes --- docs/conf.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.in b/docs/conf.in index f904d1c26b..f3f89602f7 100644 --- a/docs/conf.in +++ b/docs/conf.in @@ -168,7 +168,7 @@ class AutoAutoSummary(Autosummary): if not include_public: include_public = [] items = [] - for name in sorted(obj.__dict__.keys()): + for name in sorted(dir(obj)): try: documenter = get_documenter(app, safe_getattr(obj, name), obj) except AttributeError: From 7d787fa5764a9fe3cc5b9f976ecf050deeadebd3 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Thu, 2 Dec 2021 22:50:14 -0600 Subject: [PATCH 101/229] Integrate Python API rst generation into CMake - Python rst files are now generated by the `Sphinx` target - `clean` cleans up the generated rst files for both dpctl and libsyclinterface. - Clean up redundancies in the `_generate_sphinx` function. - Python is required for documentation generation. --- docs/CMakeLists.txt | 109 ++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 48d7cf905b..82e8e1a0e1 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -29,7 +29,7 @@ function(_setup_doxygen) if(DPCTL_ENABLE_DOXYGEN_HTML) set(GENERATE_HTML "YES") endif() - set(DOXYGEN_INPUT_DIR ../libsyclinterface/include) + set(DOXYGEN_INPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libsyclinterface/include) set(DOXYGEN_OUTPUT_DIR ${DOC_OUTPUT_DIR}/doxygen) set(DOXYGEN_INDEX_FILE ${DOXYGEN_OUTPUT_DIR}/xml/index.xml) set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) @@ -62,20 +62,23 @@ endfunction() function(_setup_doxyrest) set(DOXYREST_OUTPUT_DIR_NAME docfiles/libsyclinterface) + # Set the DOXYREST_OUTPUT_DIR variable in both current and parent scope. + # The variable is used by _setup_sphinx when generating the conf.py file. set(DOXYREST_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DOXYREST_OUTPUT_DIR_NAME} PARENT_SCOPE ) set(DOXYREST_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DOXYREST_OUTPUT_DIR_NAME} - ) set(DOXYREST_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/doxyrest-config.lua.in) set(DOXYREST_CONFIG_OUT ${CMAKE_CURRENT_SOURCE_DIR}/doxyrest-config.lua) set(DOXYREST_OUTPUT ${DOXYREST_OUTPUT_DIR}/index.rst) set(DOXYGEN_OUTPUT_DIR ${DOC_OUTPUT_DIR}/doxygen) + configure_file(${DOXYREST_CONFIG_IN} ${DOXYREST_CONFIG_OUT} @ONLY) configure_file(${INDEX_DOXYREST_IN} ${INDEX_OUT} @ONLY) + add_custom_command( OUTPUT ${DOXYREST_OUTPUT} COMMAND @@ -83,9 +86,9 @@ function(_setup_doxyrest) ${DOXYREST_CONFIG_OUT} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS - # Other docs files that can be edited manually - ${INDEX_OUT} - ${DOXYGEN_INDEX_FILE} + # Other docs files that can be edited manually + ${INDEX_OUT} + ${DOXYGEN_INDEX_FILE} MAIN_DEPENDENCY ${DOXYREST_CONFIG_OUT} ${DOXYREST_CONFIG_IN} COMMENT "Generating Doxyrest documentation" ) @@ -98,58 +101,58 @@ function(_setup_doxyrest) endfunction() function(_setup_sphinx) + set(GENERATE_RST_OUTPUT_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/docfiles/dpctl + ) set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}) set(SPHINX_OUTPUT_DIR ${DOC_OUTPUT_DIR}/docs) set(SPHINX_INDEX_FILE ${SPHINX_OUTPUT_DIR}/index.html) set(SPHINX_CONF_IN ${SPHINX_SOURCE}/conf.in) set(SPHINX_CONF_OUT ${SPHINX_SOURCE}/conf.py) - # Only regenerate Sphinx when: - # - Doxygen has rerun - # - Our doc files have been updated - # - The Sphinx config has been updated + set(DPCTL_PYAPI_RST_FILE ${GENERATE_RST_OUTPUT_DIR}/dpctl_pyapi.rst) + if(DPCTL_ENABLE_DOXYREST) - add_custom_command( - OUTPUT ${SPHINX_INDEX_FILE} - COMMAND - ${SPHINX_EXECUTABLE} -b html - ${SPHINX_SOURCE} - ${SPHINX_OUTPUT_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS - # Other docs files that can be edited manually - ${CMAKE_CURRENT_SOURCE_DIR}/index.rst - ${DOXYGEN_INDEX_FILE} - MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} - COMMENT "Generating Sphinx documentation" - ) - # Target to generate Sphinx - add_custom_target( - Sphinx - ALL - DEPENDS Doxyrest ${SPHINX_INDEX_FILE} - ) + set(DEPEND_ON_DOXYREST "Doxyrest") else() configure_file(${INDEX_NO_DOXYREST_IN} ${INDEX_OUT} @ONLY) - add_custom_command( - OUTPUT ${SPHINX_INDEX_FILE} - COMMAND - ${SPHINX_EXECUTABLE} -b html - ${SPHINX_SOURCE} - ${SPHINX_OUTPUT_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS - # Other docs files that can be edited manually - ${CMAKE_CURRENT_SOURCE_DIR}/index.rst - MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} - COMMENT "Generating Sphinx documentation" - ) - # Target to generate Sphinx - add_custom_target( - Sphinx - ALL - DEPENDS ${SPHINX_INDEX_FILE} - ) endif() + + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/generate_rst.py" + ${CMAKE_CURRENT_BINARY_DIR} + ) + # A custom command to generate the Python API rst files + add_custom_command( + OUTPUT ${DPCTL_PYAPI_RST_FILE} + COMMAND ${CMAKE_COMMAND} -E make_directory ${GENERATE_RST_OUTPUT_DIR} + COMMAND + "${Python_EXECUTABLE}" + "${CMAKE_CURRENT_BINARY_DIR}/generate_rst.py" + --dir "${GENERATE_RST_OUTPUT_DIR}" + --module "dpctl" + COMMENT "Generating RST files for Python API of dpctl" + ) + add_custom_command( + OUTPUT ${SPHINX_INDEX_FILE} + COMMAND + ${SPHINX_EXECUTABLE} -b html + ${SPHINX_SOURCE} + ${SPHINX_OUTPUT_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/index.rst + MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} + COMMENT "Generating Sphinx documentation" + ) + # Target to generate Sphinx. Note that the order of the dependencies is + # important, we want the rst files to generate prior to sphinx build. + add_custom_target( + Sphinx + ALL + DEPENDS + ${DEPEND_ON_DOXYREST} + ${DPCTL_PYAPI_RST_FILE} + ${SPHINX_INDEX_FILE} + ) # Create a conf.py by replacing variables inside @@ with the current values configure_file(${SPHINX_CONF_IN} ${SPHINX_CONF_OUT} @ONLY) endfunction() @@ -185,6 +188,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) find_package(Git) find_package(Sphinx REQUIRED) find_package(Doxygen REQUIRED) +find_package(Python REQUIRED) if (DPCTL_ENABLE_DOXYREST) find_package(Lua REQUIRED) @@ -208,5 +212,12 @@ _setup_doxygen() if(DPCTL_ENABLE_DOXYREST) _setup_doxyrest() endif() -_generate_rst() _setup_sphinx() + +set_property( + DIRECTORY + PROPERTY + ADDITIONAL_CLEAN_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/docfiles/dpctl" + "${CMAKE_CURRENT_SOURCE_DIR}/docfiles/libsyclinterface" +) From b60d928b5f88e57a98b5f6c9f9b7f0e53ef578f8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 7 Dec 2021 13:08:26 -0600 Subject: [PATCH 102/229] Adds docstring to event_status_type enum (#715) --- dpctl/enum_types.py | 48 ++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/dpctl/enum_types.py b/dpctl/enum_types.py index 5add734167..bdf95959c0 100644 --- a/dpctl/enum_types.py +++ b/dpctl/enum_types.py @@ -33,14 +33,19 @@ class device_type(Enum): """ An enumeration of supported SYCL device types. - ================== ============ - Device type Enum value - ================== ============ - gpu 1 - cpu 2 - accelerator 3 - host 4 - ================== ============ + :Example: + .. code-block:: python + + import dpctl + + # filter GPU devices amongst available SYCL devices + gpu_devs = [ + d for d in dpctl.get_devices() if ( + d.device_type == dpctl.device_type.gpu + ) ] + + # alternatively, get GPU devices directly + gpu_devs2 = dpctl.get_devices(device_type=dpctl.device_type.gpu) """ all = auto() @@ -56,15 +61,15 @@ class backend_type(Enum): """ An enumeration of supported SYCL backends. - ================== ============ - Name of backend Enum value - ================== ============ - opencl 1 - level_zero 2 - cuda 3 - host 4 - ================== ============ + :Example: + .. code-block:: python + + import dpctl + # create a SYCL device with OpenCL backend using filter selector + d = dpctl.SyclDevice("opencl") + print(d.backend) + # Possible output: """ all = auto() @@ -75,6 +80,17 @@ class backend_type(Enum): class event_status_type(Enum): + """ + An enumeration of SYCL event states. + + :Example: + .. code-block:: python + + import dpctl + ev = dpctl.SyclEvent() + print(ev.execution_status ) + # Possible output: + """ unknown_status = auto() submitted = auto() From 4dbc45ab853cfcb32bb211cb00542d185cf53044 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 7 Dec 2021 18:24:24 +0300 Subject: [PATCH 103/229] Fix handling of verbosity conditions in error_handler Renamed get_requested_level to requested_verbosity_level Utility function requested_verbosity_level moved to anonymous namespace --- .../helper/source/dpctl_error_handlers.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp index 33beb7bb9f..3755608cf1 100644 --- a/libsyclinterface/helper/source/dpctl_error_handlers.cpp +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -40,23 +40,26 @@ void DPCTL_AsyncErrorHandler::operator()( } } -inline int get_requested_level(void) +namespace +{ +int requested_verbosity_level(void) { int requested_level = 0; const char *verbose = std::getenv("DPCTL_VERBOSITY"); if (verbose) { - if (!strncmp(verbose, "none", 4)) + if (!std::strncmp(verbose, "none", 4)) requested_level = error_level::none; - else if (!strncmp(verbose, "error", 5)) + else if (!std::strncmp(verbose, "error", 5)) requested_level = error_level::error; - else if (!strncmp(verbose, "warning", 7)) + else if (!std::strncmp(verbose, "warning", 7)) requested_level = error_level::warning; } return requested_level; } +} // namespace void error_handler(const std::exception &e, const char *file_name, @@ -64,10 +67,10 @@ void error_handler(const std::exception &e, int line_num, error_level error_type) { - int requested_level = get_requested_level(); + int requested_level = requested_verbosity_level(); int error_level = static_cast(error_type); - if (requested_level <= error_level) { + if (requested_level >= error_level) { std::cerr << e.what() << " in " << func_name << " at " << file_name << ":" << line_num << std::endl; } @@ -79,10 +82,10 @@ void error_handler(const std::string &what, int line_num, error_level error_type) { + int requested_level = requested_verbosity_level(); int error_level = static_cast(error_type); - int requested_level = get_requested_level(); - if (requested_level <= error_level) { + if (requested_level >= error_level) { std::cerr << what << " In " << func_name << " at " << file_name << ":" << line_num << std::endl; } From adaa98d18868cc1f09c8f5f7e50244b9b71d9e3b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 7 Dec 2021 16:07:59 -0600 Subject: [PATCH 104/229] Tests should be run with DPCTL_VERBOSITY=warning --- libsyclinterface/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsyclinterface/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt index 1ecb4e847d..c4926ffa2d 100644 --- a/libsyclinterface/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -59,7 +59,7 @@ if(DPCTL_GENERATE_COVERAGE) ) add_custom_target(llvm-cov COMMAND ${CMAKE_MAKE_PROGRAM} dpctl_c_api_tests - COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=error ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests + COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=warning ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests COMMAND ${LLVMProfdata_EXE} merge -sparse default.profraw From 7fecbd147403366f5e729e17813281863b593514 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 8 Dec 2021 18:46:56 -0600 Subject: [PATCH 105/229] Merge pull request #717 from IntelPython/fix/broken-doxyrest-links Fix broken links in the libsyclinterface html pages. --- docs/Doxyfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 934262a3e6..49f8ec208b 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -562,7 +562,7 @@ INTERNAL_DOCS = NO # (including Cygwin) ands Mac users are advised to set this option to NO. # The default value is: system dependent. -CASE_SENSE_NAMES = YES +CASE_SENSE_NAMES = NO # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES, the From 6bf0dbedc056969af1ae4e656dfad56d27f35b6f Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Thu, 9 Dec 2021 23:10:08 -0600 Subject: [PATCH 106/229] Fix warnings thrown during Sphinx generation.' (#720) --- docs/docfiles/urls.rst | 1 - docs/generate_rst.py | 6 ++++++ dpctl/__init__.py | 4 ++-- dpctl/_sycl_event.pyx | 4 +++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/docfiles/urls.rst b/docs/docfiles/urls.rst index d78c1ba703..25bbf743c8 100644 --- a/docs/docfiles/urls.rst +++ b/docs/docfiles/urls.rst @@ -1,4 +1,3 @@ -.. __urls: .. _buffer protocol: https://docs.python.org/3/c-api/buffer.html .. _Data API: https://data-apis.github.io/array-api/latest/ diff --git a/docs/generate_rst.py b/docs/generate_rst.py index c11c2b38fe..4e3d39d17e 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -527,11 +527,17 @@ def _write_function_groups_summary(o, mod, groups): _write_line(output, ".. currentmodule:: " + module) _write_empty_line(output) _write_line(output, ".. automodule:: " + module) + _write_empty_line(output) _write_submodules_summary_table(output, mod) + _write_empty_line(output) _write_classes_summary_table(output, mod) + _write_empty_line(output) _write_function_groups_summary(output, mod, _group_functions(mod)) + _write_empty_line(output) _write_enums_summary_table(output, mod) + _write_empty_line(output) _write_exceptions_summary_table(output, mod) + _write_empty_line(output) _write_include_urls(output) return output.getvalue() diff --git a/dpctl/__init__.py b/dpctl/__init__.py index 0052465b15..e3ce7907c8 100644 --- a/dpctl/__init__.py +++ b/dpctl/__init__.py @@ -133,8 +133,8 @@ def get_include(): - """ - Return the directory that contains the dpctl *.h header files. + r""" + Return the directory that contains the dpctl \*.h header files. Extension modules that need to be compiled against dpctl should use this function to locate the appropriate include directory. diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index 1d02450775..5dfb8ee10b 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -259,13 +259,15 @@ cdef class SyclEvent(_SyclEvent): **SyclEventRef**. The ownership of the pointer inside the capsule is passed to the caller, and pointer is deleted when the capsule goes out of scope. + Returns: :class:`pycapsule`: A capsule object storing a copy of the - ``cl::sycl::event`` pointer belonging to thus + ``cl::sycl::event`` pointer belonging to a :class:`dpctl.SyclEvent` instance. Raises: ValueError: If the ``DPCTLEvent_Copy`` fails to copy the ``cl::sycl::event`` pointer. + """ cdef DPCTLSyclEventRef ERef = NULL ERef = DPCTLEvent_Copy(self._event_ref) From b7a15ed9a341812c06415f7fcf63466f82ece591 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Fri, 10 Dec 2021 18:40:13 -0600 Subject: [PATCH 107/229] Add a special case to document device factory functions. (#721) - _sycl_device_factory consists of device selection and other functions. Added a check to list all device selections functions as a group and all other functions as "Others". --- docs/generate_rst.py | 58 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/generate_rst.py b/docs/generate_rst.py index 4e3d39d17e..b2990fef2b 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -39,7 +39,6 @@ # Dictionary mapping internal module names to a readable string. so that we # can use the module name to logically group functions. function_groups = { - "dpctl._sycl_device_factory": "Device Selection Functions", "dpctl._device_selection": "Device Selection Functions", "dpctl._sycl_queue_manager": "Queue Management Functions", "dpctl.tensor._ctors": "Array Construction", @@ -237,13 +236,26 @@ def _group_functions(mod): obj, ] else: - try: - flist = groups["Other Functions"] - flist.append(obj) - except KeyError: - groups["Other Functions"] = [ - obj, - ] + # Special case for _sycl_device_factory + if ( + obj.__module__ == "dpctl._sycl_device_factory" + and "select_" in obj.__name__ + ): + try: + flist = groups["Device Selection Functions"] + flist.append(obj) + except KeyError: + groups["Device Selection Functions"] = [ + obj, + ] + else: + try: + flist = groups["Other Functions"] + flist.append(obj) + except KeyError: + groups["Other Functions"] = [ + obj, + ] return groups @@ -455,6 +467,8 @@ def _write_classes_summary_table(o, mod): classes.append(cls) class_names.append(mem_tup[0]) if classes: + _write_line(o, ".. _" + mod.__name__.lower() + "_classes:") + _write_empty_line(o) _write_underlined(o, "Classes", "-") _write_empty_line(o) _write_hidden_toc(output, class_names) @@ -511,11 +525,35 @@ def _write_functions_summary_table(o, mod, fnobj_list): _write_empty_line(o) def _write_function_groups_summary(o, mod, groups): + for group in groups: + if group != "Other Functions": + _write_line( + o, + ".. _" + + mod.__name__.lower() + + "_" + + group.lower().replace(" ", "_") + + ":", + ) + _write_empty_line(o) + _write_underlined(o, group, "-") + _write_empty_line(o) + _write_functions_summary_table(o, mod, groups[group]) + + # We want to write "Other Functions" in the end always + try: + other_fns = groups["Other Functions"] + _write_line( + o, + ".. _" + mod.__name__.lower() + "_other_functions:", + ) _write_empty_line(o) - _write_underlined(o, group, "-") + _write_underlined(o, "Other Functions", "-") _write_empty_line(o) - _write_functions_summary_table(o, mod, groups[group]) + _write_functions_summary_table(o, mod, other_fns) + except KeyError: + pass mod = _get_module(module) From 4580c7132ead07016eb212085527a8e76d9bab65 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 13:31:02 -0600 Subject: [PATCH 108/229] Fixes #723 Restores parity in performance of two scenarios in time_copy.py script ``` (idp_2021.4) [13:33:21 ansatnuc04 python]$ python time_copy.py Wall time: 0.0004440806806087494 sec. Device time: 9.926800000000001e-05 sec. Wall time: 0.0006928546354174614 sec. Device time: 0.000150562 sec. ``` --- dpctl/tensor/_copy_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index 13866fc973..c49862d70f 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -117,6 +117,12 @@ def _copy_from_numpy_into(dst, np_ary): if not isinstance(np_ary, np.ndarray): raise TypeError("Expected numpy.ndarray, got {}".format(type(np_ary))) src_ary = np.broadcast_to(np.asarray(np_ary, dtype=dst.dtype), dst.shape) + if src_ary.size and (dst.flags & 1) and src_ary.flags["C"]: + dpm.as_usm_memory(dst).copy_from_host(src_ary.reshape((-1,)).view("u1")) + return + if src_ary.size and (dst.flags & 2) and src_ary.flags["F"]: + dpm.as_usm_memory(dst).copy_from_host(src_ary.reshape((-1,)).view("u1")) + return for i in range(dst.size): mi = np.unravel_index(i, dst.shape) host_buf = np.array(src_ary[mi], ndmin=1).view("u1") From cb07c38c2128ea975e702e073a91c9885f3e7a9a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 16:54:40 -0600 Subject: [PATCH 109/229] Added tests to cover branches in copy from Numpy array to usm array --- dpctl/tests/test_tensor_asarray.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py index d8c6f9d7ce..f4e3d77bfb 100644 --- a/dpctl/tests/test_tensor_asarray.py +++ b/dpctl/tests/test_tensor_asarray.py @@ -67,7 +67,19 @@ def test_asarray_from_numpy(): Xnp = np.arange(10) Y = dpt.asarray(Xnp, usm_type="device") assert type(Y) is dpt.usm_ndarray - assert Y.shape == (10,) + assert Y.shape == Xnp.shape + assert Y.dtype == Xnp.dtype + # Fortan contiguous case + Xnp = np.array([[1, 2, 3], [4, 5, 6]], dtype="f4", order="F") + Y = dpt.asarray(Xnp, usm_type="shared") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == Xnp.shape + assert Y.dtype == Xnp.dtype + # general strided case + Xnp = np.array([[1, 2, 3], [4, 5, 6]], dtype="i8") + Y = dpt.asarray(Xnp[::-1, ::-1], usm_type="host") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == Xnp.shape assert Y.dtype == Xnp.dtype From 965fa3b19c5d4fb0bb6944901ff00d2329764098 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 14 Dec 2021 16:16:46 -0600 Subject: [PATCH 110/229] Added an example showing filter string usage. (#725) - Use try..except in examples to make sure we do not crash on a system where needed drivers are not installed. --- examples/python/device_selection.py | 12 +++--- examples/python/filter_selection.py | 64 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 examples/python/filter_selection.py diff --git a/examples/python/device_selection.py b/examples/python/device_selection.py index 0acafe6a17..b7ee98dc1a 100644 --- a/examples/python/device_selection.py +++ b/examples/python/device_selection.py @@ -54,11 +54,13 @@ def create_gpu_device(): SYCL_DEVICE_FILTER, which determines SYCL devices seen by the SYCL runtime. """ - d1 = dpctl.SyclDevice("gpu") - d2 = dpctl.select_gpu_device() - assert d1 == d2 - print_device(d1) - return d1 + try: + d1 = dpctl.SyclDevice("gpu") + d2 = dpctl.select_gpu_device() + assert d1 == d2 + print_device(d1) + except ValueError: + print("A GPU device is not available on the system") def create_gpu_device_if_present(): diff --git a/examples/python/filter_selection.py b/examples/python/filter_selection.py new file mode 100644 index 0000000000..e351d70323 --- /dev/null +++ b/examples/python/filter_selection.py @@ -0,0 +1,64 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Examples illustrating SYCL device selection using filter strings. +""" + +import dpctl + + +def print_device(d): + "Display information about given device argument." + if type(d) is not dpctl.SyclDevice: + raise ValueError + print("Name: ", d.name) + print("Vendor: ", d.vendor) + print("Driver version: ", d.driver_version) + print("Backend: ", d.backend) + print("Max EU: ", d.max_compute_units) + + +def select_using_filter(): + """ + Demonstrate the usage of a filter string to create a SyclDevice. + + """ + try: + d1 = dpctl.SyclDevice("cpu") + print_device(d1) + except ValueError: + print("A CPU type device is not available on the system") + + try: + d1 = dpctl.SyclDevice("opencl:cpu:0") + print_device(d1) + except ValueError: + print("An OpenCL CPU driver needs to be installed on the system") + + d1 = dpctl.SyclDevice("0") + print_device(d1) + + try: + d1 = dpctl.SyclDevice("gpu") + print_device(d1) + except ValueError: + print("A GPU type device is not available on the system") + + +if __name__ == "__main__": + import _runner as runner + + runner.run_examples("Filter selection examples for dpctl.", globals()) From a96214cb9bdfd71c3d3f0c73dc681ad821641781 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 14 Dec 2021 18:51:35 -0600 Subject: [PATCH 111/229] Add example for select_device_with_aspects. (#726) --- examples/python/device_selection.py | 29 +++++++++++++++-------------- examples/python/filter_selection.py | 19 ++++--------------- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/examples/python/device_selection.py b/examples/python/device_selection.py index b7ee98dc1a..55b65b9ea5 100644 --- a/examples/python/device_selection.py +++ b/examples/python/device_selection.py @@ -20,17 +20,6 @@ import dpctl -def print_device(d): - "Display information about given device argument." - if type(d) is not dpctl.SyclDevice: - raise ValueError - print("Name: ", d.name) - print("Vendor: ", d.vendor) - print("Driver version: ", d.driver_version) - print("Backend: ", d.backend) - print("Max EU: ", d.max_compute_units) - - def create_default_device(): """ Create default SyclDevice using `cl::sycl::default_selector`. @@ -42,7 +31,7 @@ def create_default_device(): d1 = dpctl.SyclDevice() d2 = dpctl.select_default_device() assert d1 == d2 - print_device(d1) + d1.print_device_info() return d1 @@ -58,7 +47,7 @@ def create_gpu_device(): d1 = dpctl.SyclDevice("gpu") d2 = dpctl.select_gpu_device() assert d1 == d2 - print_device(d1) + d1.print_device_info() except ValueError: print("A GPU device is not available on the system") @@ -96,12 +85,24 @@ def custom_select_device(): max_score = d.default_selector_score selected_dev = d if selected_dev: - print_device(selected_dev) + selected_dev.print_device_info() else: print("No device with half-precision support is available.") return selected_dev +def create_device_with_aspects(): + """ + Programmatically select a device based on specific set of aspects. + + Demonstrate the usage of :func:`dpctl.select_device_with_aspects()`. + """ + dev = dpctl.select_device_with_aspects( + required_aspects=["fp64", "gpu", "usm_shared_allocations"] + ) + dev.print_device_info() + + if __name__ == "__main__": import _runner as runner diff --git a/examples/python/filter_selection.py b/examples/python/filter_selection.py index e351d70323..af6dc91fc0 100644 --- a/examples/python/filter_selection.py +++ b/examples/python/filter_selection.py @@ -20,17 +20,6 @@ import dpctl -def print_device(d): - "Display information about given device argument." - if type(d) is not dpctl.SyclDevice: - raise ValueError - print("Name: ", d.name) - print("Vendor: ", d.vendor) - print("Driver version: ", d.driver_version) - print("Backend: ", d.backend) - print("Max EU: ", d.max_compute_units) - - def select_using_filter(): """ Demonstrate the usage of a filter string to create a SyclDevice. @@ -38,22 +27,22 @@ def select_using_filter(): """ try: d1 = dpctl.SyclDevice("cpu") - print_device(d1) + d1.print_device_info() except ValueError: print("A CPU type device is not available on the system") try: d1 = dpctl.SyclDevice("opencl:cpu:0") - print_device(d1) + d1.print_device_info() except ValueError: print("An OpenCL CPU driver needs to be installed on the system") d1 = dpctl.SyclDevice("0") - print_device(d1) + d1.print_device_info() try: d1 = dpctl.SyclDevice("gpu") - print_device(d1) + d1.print_device_info() except ValueError: print("A GPU type device is not available on the system") From cd0d7c7d3a3def9f99bc1b37834ad9df56c4d8c4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Dec 2021 07:29:36 -0600 Subject: [PATCH 112/229] A step added to build/run examples Adjust triggers to avoid duplicate jobs on push to branches with a PR Activate and stack activate in shell session where examples are built Only activate build_env for building packages, deactivate it before running example Make sure to set OCL_ICD_FILENAMES before running examples --- .github/workflows/conda-package.yml | 116 +++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 86d9a025ed..f4c15abfb2 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -1,6 +1,10 @@ name: Conda package -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: env: PACKAGE_NAME: dpctl @@ -115,6 +119,7 @@ jobs: - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH - name: Install conda-build + # Needed to be able to run conda index run: conda install conda-build - name: Create conda channel run: | @@ -178,6 +183,7 @@ jobs: auto-activate-base: true activate-environment: "" - name: Install conda-build + # Needed to be able to run conda index run: conda install conda-build - name: Create conda channel run: | @@ -263,3 +269,111 @@ jobs: run: | conda install anaconda-client anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.tar.bz2 + + test_examples_linux: + needs: build_linux + runs-on: ${{ matrix.runner }} + strategy: + matrix: + python: [3.8] + experimental: [false] + runner: [ubuntu-latest] + continue-on-error: ${{ matrix.experimental }} + env: + CHANNELS: -c intel -c defaults --override-channels + + steps: + - name: Install conda-build + # Needed to be able to run conda index + run: conda install conda-build python=${{ matrix.python }} + - name: Checkout dpctl repo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + - name: Add conda to system path + run: echo $CONDA/bin >> $GITHUB_PATH + - name: Create conda channel + run: | + mkdir -p $GITHUB_WORKSPACE/channel/linux-64 + mv ${PACKAGE_NAME}-*.tar.bz2 $GITHUB_WORKSPACE/channel/linux-64 + conda index $GITHUB_WORKSPACE/channel + # Test channel + conda search $PACKAGE_NAME -c $GITHUB_WORKSPACE/channel --override-channels + - name: Collect dependencies + run: | + CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" + conda install $PACKAGE_NAME python=${{ matrix.python }} $CHANNELS --only-deps --dry-run > lockfile + - name: Set pkgs_dirs + run: | + echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc + - name: Cache conda packages + uses: actions/cache@v2 + env: + CACHE_NUMBER: 0 # Increase to reset cache + with: + path: ~/.conda/pkgs + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-${{hashFiles('lockfile') }} + restore-keys: | + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}- + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- + - name: Install dpctl + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + conda activate + CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" + conda install -y $PACKAGE_NAME pytest python=${{ matrix.python }} $CHANNELS + # Test installed packages + conda list + - name: Install example requirements + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + conda install -y pybind11 cython + conda install -y -c intel mkl-dpcpp mkl-devel-dpcpp numba-dppy + conda create -y -n build_env -c intel dpcpp_linux-64 + - name: Build and run examples with native extensions + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + conda activate + cd examples/pybind11 + export CC=dpcpp + export CXX=dpcpp + for d in $(ls) + do + pushd $d + conda activate --stack build_env + python setup.py build_ext --inplace || exit 1 + conda deactivate + python example.py + popd + done + cd ../cython + for d in $(ls) + do + pushd $d + conda activate --stack build_env + python setup.py build_ext --inplace || exit 1 + conda deactivate + python run.py + popd + done + - name: Run Python examples + shell: bash -l {0} + run: | + cd examples/python + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + for script in $(find . \( -not -name "_*" -and -name "*.py" \)) + do + echo "Executing ${script}" + python ${script} || exit 1 + done From a1a93af3ef8ddf615e83ebd21bce755c436ba66e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 18:01:56 -0600 Subject: [PATCH 113/229] Use default selected queue, rather than gpu selected queue --- examples/pybind11/external_usm_allocation/example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pybind11/external_usm_allocation/example.py b/examples/pybind11/external_usm_allocation/example.py index 522f822a36..4455616146 100644 --- a/examples/pybind11/external_usm_allocation/example.py +++ b/examples/pybind11/external_usm_allocation/example.py @@ -22,7 +22,7 @@ import dpctl import dpctl.memory as dpm -q = dpctl.SyclQueue("gpu") +q = dpctl.SyclQueue() matr = eua.DMatrix(q, 5, 5) print(matr) From 7342189d73e5c1ecfaae31f776c2c693bc9add32 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 18:15:05 -0600 Subject: [PATCH 114/229] Added C++ and language level directive to .pyx files Added $CONDA_PREFIX/include to include dirs in Extensions using MKL --- examples/cython/sycl_buffer/_buffer_example.pyx | 3 +++ examples/cython/sycl_buffer/setup.py | 10 +++++++++- .../cython/sycl_direct_linkage/_buffer_example.pyx | 3 +++ examples/cython/sycl_direct_linkage/setup.py | 10 +++++++++- examples/cython/usm_memory/setup.py | 10 +++++++++- 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/examples/cython/sycl_buffer/_buffer_example.pyx b/examples/cython/sycl_buffer/_buffer_example.pyx index 2737dbcc72..2f442d9388 100644 --- a/examples/cython/sycl_buffer/_buffer_example.pyx +++ b/examples/cython/sycl_buffer/_buffer_example.pyx @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# distutils: language = c++ +# cython: language_level=3 + cimport numpy as cnp import numpy as np diff --git a/examples/cython/sycl_buffer/setup.py b/examples/cython/sycl_buffer/setup.py index 8efce46d7a..3e2d98390f 100644 --- a/examples/cython/sycl_buffer/setup.py +++ b/examples/cython/sycl_buffer/setup.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path +import sysconfig + import numpy as np from setuptools import Extension, setup @@ -39,7 +42,12 @@ "_buffer_example.pyx", "use_sycl_buffer.cpp", ], - include_dirs=[".", np.get_include(), dpctl.get_include()], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], libraries=["sycl"] + [ "mkl_sycl", diff --git a/examples/cython/sycl_direct_linkage/_buffer_example.pyx b/examples/cython/sycl_direct_linkage/_buffer_example.pyx index 49d6cbd95e..effff5117c 100644 --- a/examples/cython/sycl_direct_linkage/_buffer_example.pyx +++ b/examples/cython/sycl_direct_linkage/_buffer_example.pyx @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# distutils: language = c++ +# cython: language_level=3 + cimport numpy as cnp import numpy as np diff --git a/examples/cython/sycl_direct_linkage/setup.py b/examples/cython/sycl_direct_linkage/setup.py index 31c1e92162..b60e358afa 100644 --- a/examples/cython/sycl_direct_linkage/setup.py +++ b/examples/cython/sycl_direct_linkage/setup.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path +import sysconfig + import numpy as np from setuptools import Extension, setup @@ -43,7 +46,12 @@ "_buffer_example.pyx", "sycl_function.cpp", ], - include_dirs=[".", np.get_include(), dpctl.get_include()], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], libraries=["sycl"] + [ "mkl_sycl", diff --git a/examples/cython/usm_memory/setup.py b/examples/cython/usm_memory/setup.py index 008370fd62..cf5e4ed122 100644 --- a/examples/cython/usm_memory/setup.py +++ b/examples/cython/usm_memory/setup.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path +import sysconfig + import numpy as np from setuptools import Extension, setup @@ -38,7 +41,12 @@ "blackscholes.pyx", "sycl_blackscholes.cpp", ], - include_dirs=[".", np.get_include(), dpctl.get_include()], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], libraries=["sycl"] + [ "mkl_sycl", From 4d92808afea7505693bcce52f136202681cd3a52 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 19:23:55 -0600 Subject: [PATCH 115/229] Use try/catch to handle failure to create a queue --- examples/python/dppy_kernel.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/python/dppy_kernel.py b/examples/python/dppy_kernel.py index cfbbc947f6..05e7dd6cfc 100644 --- a/examples/python/dppy_kernel.py +++ b/examples/python/dppy_kernel.py @@ -44,7 +44,14 @@ def dppy_gemm(a, b, c): b = np.array(np.random.random(X * X), dtype=np.float32).reshape(X, X) c = np.ones_like(a).reshape(X, X) -q = dpctl.SyclQueue("opencl:gpu", property="enable_profiling") +try: + q = dpctl.SyclQueue("opencl:gpu", property="enable_profiling") +except dpctl.SyclQueueCreationError: + print( + "Skipping the example, as dpctl.SyclQueue targeting " + "opencl:gpu device could not be created" + ) + exit(0) timer = SyclTimer(time_scale=1) with dpctl.device_context(q): with timer(q): From 9842e3c9702dffc522530c7b007188447393dbff Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 19:25:32 -0600 Subject: [PATCH 116/229] Use try/except to handle failure to create SyclQueue --- examples/cython/sycl_buffer/run.py | 31 +++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/examples/cython/sycl_buffer/run.py b/examples/cython/sycl_buffer/run.py index a82a6671a0..a66ae60dea 100644 --- a/examples/cython/sycl_buffer/run.py +++ b/examples/cython/sycl_buffer/run.py @@ -23,18 +23,31 @@ print("Result computed by NumPy") print(X.sum(axis=0)) -print("Result computed by SYCL extension using default offloading target") -print(sb.columnwise_total(X)) +try: + res = sb.columnwise_total(X) + print("Result computed by SYCL extension using default offloading target") + print(res) +except dpctl.SyclQueueCreationError: + print( + "Could not create SyclQueue for default selected device. Nothing to do." + ) + exit(0) print("") # controlling where to offload -q = dpctl.SyclQueue("opencl:gpu") -print("Running on: ", q.sycl_device.name) -print(sb.columnwise_total(X, queue=q)) - -q = dpctl.SyclQueue("opencl:cpu") -print("Running on: ", q.sycl_device.name) -print(sb.columnwise_total(X, queue=q)) +try: + q = dpctl.SyclQueue("opencl:gpu") + print("Running on: ", q.sycl_device.name) + print(sb.columnwise_total(X, queue=q)) +except dpctl.SyclQueueCreationError: + print("Not running onf opencl:gpu, queue could not be created") + +try: + q = dpctl.SyclQueue("opencl:cpu") + print("Running on: ", q.sycl_device.name) + print(sb.columnwise_total(X, queue=q)) +except dpctl.SyclQueueCreationError: + print("Not running onf opencl:cpu, queue could not be created") From db23331b86df856864aa74cffbf85f8d228ef101 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 13 Dec 2021 19:36:01 -0600 Subject: [PATCH 117/229] Make example capable of running for different SYCL platform configurations --- examples/cython/usm_memory/run.py | 99 ++++++++++++++----------------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/examples/cython/usm_memory/run.py b/examples/cython/usm_memory/run.py index 2b676288dd..720c580250 100644 --- a/examples/cython/usm_memory/run.py +++ b/examples/cython/usm_memory/run.py @@ -64,59 +64,48 @@ def gen_option_params( n_opts = 3 * 10 ** 6 # compute on CPU sycl device -cpu_q = dpctl.SyclQueue("opencl:cpu:0") -opts1 = gen_option_params( - n_opts, - 20.0, - 30.0, - 22.0, - 29.0, - 18.0, - 24.0, - 0.01, - 0.05, - 0.01, - 0.05, - "d", - queue=cpu_q, -) - -gpu_q = dpctl.SyclQueue("level_zero:gpu:0") -opts2 = gen_option_params( - n_opts, - 20.0, - 30.0, - 22.0, - 29.0, - 18.0, - 24.0, - 0.01, - 0.05, - 0.01, - 0.05, - "d", - queue=gpu_q, -) - -cpu_times = [] -gpu_times = [] -for _ in range(5): - - t0 = timeit.default_timer() - X1 = bs.black_scholes_price(opts1, queue=cpu_q) - t1 = timeit.default_timer() - cpu_times.append(t1 - t0) - - # compute on GPU sycl device - - t0 = timeit.default_timer() - X2 = bs.black_scholes_price(opts2, queue=gpu_q) - t1 = timeit.default_timer() - gpu_times.append(t1 - t0) - -print("Using : {}".format(cpu_q.sycl_device.name)) -print("Wall times : {}".format(cpu_times)) - -print("Using : {}".format(gpu_q.sycl_device.name)) -print("Wall times : {}".format(gpu_times)) +queues = [] +for filter_str in ["cpu", "gpu"]: + try: + q = dpctl.SyclQueue(filter_str) + queues.append(q) + except dpctl.SyclQueueCreationError: + continue + +if not queues: + print("No queues could not created, nothing to do.") + exit(0) + +opt_params_list = [] +for q in queues: + opt_params = gen_option_params( + n_opts, + 20.0, + 30.0, + 22.0, + 29.0, + 18.0, + 24.0, + 0.01, + 0.05, + 0.01, + 0.05, + "d", + queue=q, + ) + opt_params_list.append(opt_params) + +times_dict = dict() +for q, params in zip(queues, opt_params_list): + times_list = [] + for _ in range(5): + t0 = timeit.default_timer() + X1 = bs.black_scholes_price(params, queue=q) + t1 = timeit.default_timer() + times_list.append(t1 - t0) + times_dict[q.name] = times_list + +for dev_name, wall_times in times_dict.items(): + print("Using : {}".format(dev_name)) + print("Wall times : {}".format(wall_times)) From 3e1dc2cba8442bb049988e6e15f01e81aea0df3a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 15 Dec 2021 09:28:29 -0600 Subject: [PATCH 118/229] In example select device with aspects do not insists on GPU to enable it running in cloud --- examples/python/device_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/device_selection.py b/examples/python/device_selection.py index 55b65b9ea5..373b9e401a 100644 --- a/examples/python/device_selection.py +++ b/examples/python/device_selection.py @@ -98,7 +98,7 @@ def create_device_with_aspects(): Demonstrate the usage of :func:`dpctl.select_device_with_aspects()`. """ dev = dpctl.select_device_with_aspects( - required_aspects=["fp64", "gpu", "usm_shared_allocations"] + required_aspects=["fp64", "usm_shared_allocations"] ) dev.print_device_info() From 56adc96ecadf1da028209906240a89ea0b5483a6 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Mon, 20 Dec 2021 23:53:58 -0600 Subject: [PATCH 119/229] Add a urls.json file to store all urls in docs. - Add a JSON file to store all urls. - Add a new extlinks_gen module to read urls from the urls.json into the extlinks dictionary read bu the sphinx.ext.extlinks module. - Use extlinks_gen inside conf.py. - Add license header and other minor changes to conf.py. --- docs/conf.in | 42 +++++++++++++++++++++++++++++++++-------- docs/docfiles/urls.json | 16 ++++++++++++++++ docs/extlinks_gen.py | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 docs/docfiles/urls.json create mode 100644 docs/extlinks_gen.py diff --git a/docs/conf.in b/docs/conf.in index f3f89602f7..64af63037e 100644 --- a/docs/conf.in +++ b/docs/conf.in @@ -1,3 +1,19 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python3 # -*- coding: utf-8 -*- @@ -5,19 +21,23 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys + from docutils.parsers.rst import directives from sphinx.ext.autosummary import Autosummary, get_documenter from sphinx.util.inspect import safe_getattr import dpctl +sys.path.insert(0, os.path.abspath(".")) + +import extlinks_gen as urlgen + # -- Project information ----------------------------------------------------- project = "Data-parallel Control (dpctl)" -copyright = "2020, Intel Corp." +copyright = "2020-21, Intel Corp." author = "Intel Corp." version = dpctl.__version__.strip(".dirty") @@ -31,13 +51,15 @@ release = dpctl.__version__.strip(".dirty") # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx.ext.coverage", + "sphinx.ext.extlinks", + "sphinx.ext.githubpages", "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinxcontrib.programoutput", ] todo_include_todos = True @@ -209,3 +231,7 @@ class AutoAutoSummary(Autosummary): def setup(app): app.add_directive("autoautosummary", AutoAutoSummary) + + +# A dictionary of urls +extlinks = urlgen.create_extlinks() diff --git a/docs/docfiles/urls.json b/docs/docfiles/urls.json new file mode 100644 index 0000000000..3e0906fc41 --- /dev/null +++ b/docs/docfiles/urls.json @@ -0,0 +1,16 @@ +{ + "dpcpp_envar": "https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md", + "numa_domain": "https://en.wikipedia.org/wiki/Non-uniform_memory_access", + "oneapi": "https://www.oneapi.io/", + "oneapi_filter_selection": "https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/FilterSelector/FilterSelector.adoc", + "sycl_aspects": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#table.device.aspect", + "sycl_context": "https://sycl.readthedocs.io/en/latest/iface/context.html", + "sycl_device": "https://sycl.readthedocs.io/en/latest/iface/device.html", + "sycl_device_info": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_device_information_descriptors", + "sycl_device_selector": "https://sycl.readthedocs.io/en/latest/iface/device-selector.html", + "sycl_event": "https://sycl.readthedocs.io/en/latest/iface/event.html", + "sycl_platform": "https://sycl.readthedocs.io/en/latest/iface/platform.html", + "sycl_queue": "https://sycl.readthedocs.io/en/latest/iface/queue.html", + "sycl_runtime_classes": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes", + "sycl_spec_2020": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html" +} diff --git a/docs/extlinks_gen.py b/docs/extlinks_gen.py new file mode 100644 index 0000000000..caa45a9e94 --- /dev/null +++ b/docs/extlinks_gen.py @@ -0,0 +1,36 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + + +def create_extlinks(): + """Reads a JSON file to create a dictionary of urls in the format supported + by the sphinx.ect.extlinks extension. + + Returns: + dict: A dictionary that is understood by the extlinks Sphinx extension. + + """ + extlinks = {} + + with open("docfiles/urls.json") as urls_json: + urls = json.load(urls_json) + for url in urls: + url_value = urls[url] + extlinks[url] = (url_value + "%s", None) + + return extlinks From c483f3870860aca2c897dad51cba4b1bce402f3d Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:37:51 -0600 Subject: [PATCH 120/229] Sphinxcontrib is needed to show program output. --- .github/workflows/generate-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 69bd24eaff..3f81a5eadc 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -49,7 +49,7 @@ jobs: if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | - pip install numpy cython setuptools sphinx sphinx_rtd_theme pydot graphviz + pip install numpy cython setuptools sphinx sphinx_rtd_theme pydot graphviz sphinxcontrib-programoutput - name: Checkout repo uses: actions/checkout@v2 with: From 330ca847cdbbebb8995fd759d4591c9e46b29922 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:38:28 -0600 Subject: [PATCH 121/229] Fix hyperlinks in doctrings. --- dpctl/__init__.py | 11 ++++++----- dpctl/_sycl_context.pyx | 2 +- dpctl/_sycl_device.pyx | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dpctl/__init__.py b/dpctl/__init__.py index e3ce7907c8..ca4ee9796f 100644 --- a/dpctl/__init__.py +++ b/dpctl/__init__.py @@ -18,11 +18,12 @@ **Data Parallel Control (dpctl)** is a Python abstraction layer over SYCL. Dpctl implements a subset of SYCL's API providing wrappers for the - SYCL runtime classes described in `Section 4.6`_ of the `SYCL 2020 spec`_. - Note that the SYCL ``device_selector`` class is not implemented, instead - there are device selection helper functions that can be used to simulate - the same behavior. Dpctl implements the ``ONEPI::filter_selector`` extension - that is included in Intel's DPC++ SYCL compiler. + SYCL runtime classes described in :sycl_runtime_classes:`Section 4.6 <>` of + the :sycl_spec_2020:`SYCL 2020 spec <>`. Note that the SYCL + ``device_selector`` class is not implemented, instead there are device + selection helper functions that can be used to simulate the same behavior. + Dpctl implements the ``ONEPI::filter_selector`` extension that is included + in Intel's DPC++ SYCL compiler. The module also includes a global SYCL queue manager. The queue manager provides convenience functions to create a global instance of diff --git a/dpctl/_sycl_context.pyx b/dpctl/_sycl_context.pyx index 56e0094582..87103d7c8a 100644 --- a/dpctl/_sycl_context.pyx +++ b/dpctl/_sycl_context.pyx @@ -86,7 +86,7 @@ cdef class _SyclContext: cdef class SyclContext(_SyclContext): """ SyclContext(arg=None) - A Python wrapper for the `sycl context`_ C++ class. + A Python wrapper for the :sycl_context:`sycl::context <>` C++ class. There are multiple ways to create a :class:`dpctl.SyclContext` object: diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index c32575b1ae..83b409c46c 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -175,13 +175,13 @@ cdef void _init_helper(_SyclDevice device, DPCTLSyclDeviceRef DRef): cdef class SyclDevice(_SyclDevice): """ SyclDevice(arg=None) - Python equivalent for cl::sycl::device class. + A Python wrapper for the :sycl_device:`sycl::device <>` C++ class. There are two ways of creating a SyclDevice instance: - by directly passing in a filter string to the class constructor. The filter string needs to conform to the - `DPC++ filter selector SYCL extension `_. + :oneapi_filter_selection:`DPC++ filter selector SYCL extension <>`. :Example: .. code-block:: python From 02c94a052039a7f3f7cf316be081f775b3c46ab4 Mon Sep 17 00:00:00 2001 From: oleksandr-pavlyk Date: Tue, 21 Dec 2021 01:49:27 -0600 Subject: [PATCH 122/229] Add a page for the dpctl user manual defining basic SYCL concepts. --- .../manual/dpctl/basic_concepts.rst | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst diff --git a/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst b/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst new file mode 100644 index 0000000000..fbda045899 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst @@ -0,0 +1,75 @@ +.. _basic_concepts: + +Basic Concepts +============== + +The section introduces the basic concepts for XPU management used by dpctl. +As dpctl is based on SYCL the concepts should be familiar to users with prior +experience with SYCL. However, users of dpctl need not have any prior experience +with SYCL and the concepts presented here should be self-sufficient. We do not +go into all the SYCL-level details here and if needed readers should refer to a +more topical SYCL reference such as the :sycl_spec_2020:`SYCL 2020 spec <>`. + +* **Heterogeneous computing** + Refers to using multiple devices in a program. + +* **Host** + Every program starts by running on a host, and most of the lines of code in + a program, in particular lines of code implementing the Python interpreter + itself, are usually for the host. Hosts are customarily CPUs. + +* **Device** + A device is an XPU connected to a host that is programmable with a specific + device driver. Different types of devices can have different architectures + (CPUs, GPUs, FPGA, ASICs, DSP), but are programmable using the same + :oneapi:`oneAPI <>` programming model. + +* **Platform** + A device driver installed on the system is termed as a platform. As multiple + devices of the same type can share the same device driver, a platform may + contain multiple devices. Note that the same physical hardware (say, a GPU) + may be reflected as two separate devices if they can be programmed by more + than one platform, *e.g.*, the same GPU hardware can be listed as an + OpenCL GPU device and a Level-Zero GPU device. + +* **Context** + A context holds the run-time information needed to operate on a device or a + group of devices from the same platform. Contexts are relatively expensive + to create and should be reused as much as possible. + +* **Queue** + A queue is needed to schedule execution of any computation, or data + copying on the device. Queue construction requires specifying a device + and a context targeting that device as well as additional properties, + such as whether profiling information should be collected or whether submitted + tasks are executed in the order in which they were submitted. + +* **Event** + An event holds information related to computation/data movement operation + scheduled for execution on a queue, such as its execution status as well + as profiling information if the queue the task was submitted to allowed + for collection of such information. Events can be used to specify task + dependencies as well as to synchronize host and devices. + +* **USM** + Unified Shared Memory (USM) refers to pointer based device memory management. + USM allocations are bound to context. In other words, a pointer representing + USM allocation can be unambiguously mapped to the data it represents only + if the associated context is known. USM allocations are accessible by + computational kernels that are executed on a device, provided that the + allocation is bound to the same context that was used to construct the queue + where the kernel was scheduled for execution. + + Depending on the capability of the device, USM allocations can be a "device" + allocation, a "shared" allocation, or a "host" allocation. A "device" + allocation is not accessible from host, while "shared" or "host" allocations + are. "Host" allocation refers to an allocation in host memory that is + accessible from a device. + + "Shared" allocations are accessible by both host and device. Runtime manages + synchronization of host's and device's view into shared allocations. Initial + placement of the shared allocations is not defined. + +* **Backend** + Refers to an implementation of :oneapi:`oneAPI <>` programming model exposed + by the underlying runtime. From d0badc74373c3c8881c8023d8e0ee1f530489421 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:51:12 -0600 Subject: [PATCH 123/229] Add new sections for dpctl user manual. - A page for the explaining device selection. - A page for defining platforms and demonstrating platform querying. --- .../manual/dpctl/device_selection.rst | 150 ++++++++++++++++++ .../user_guides/manual/dpctl/platforms.rst | 35 ++++ 2 files changed, 185 insertions(+) create mode 100644 docs/docfiles/user_guides/manual/dpctl/device_selection.rst create mode 100644 docs/docfiles/user_guides/manual/dpctl/platforms.rst diff --git a/docs/docfiles/user_guides/manual/dpctl/device_selection.rst b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst new file mode 100644 index 0000000000..75aa7cc18f --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst @@ -0,0 +1,150 @@ +.. _device_selection: + +################ +Device Selection +################ + +Device selection refers to programmatically selecting a single device from +the set of :ref:`devices ` available on the system. + +Selecting a Specific Type of Device +----------------------------------- + +If a user needs to select a specific type of device such as a GPU, they can +directly use one of the helper functions included inside dpctl. Dpctl includes +:ref:`helper functions ` for selecting +a ``host``, a ``cpu``, a ``gpu``, an ``accelerator``, or the ``default`` device. +These functions are analogous to SYCL's built-in +:sycl_device_selector:`sycl::device_selector <>` classes. The scoring and +selection of a specific device when multiple devices of the same type are +available on a system is deferred to the underlying SYCL runtime. + +The example :ref:`fig-gpu-device-selection` shows the usage of the +:func:`dpctl.select_gpu_device()` device selection function. In case multiple +GPU devices are available, only one is returned based on the underlying scoring +logic inside the SYCL runtime. If the selection function was unable to select a +device a ``ValueError`` is raised. + +.. _fig-gpu-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 38-52 + :caption: Selecting a GPU Device + :linenos: + +A possible output for the example :ref:`fig-gpu-device-selection` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_gpu_device + +Selecting a Device Using a Filter String +---------------------------------------- + +Along with using the default device selection functions, a more explicit way of +device selection involves the use of *filter strings* (refer +:oneapi_filter_selection:`oneAPI filter selection extension <>`). The example +:ref:`fig-gpu-device-selection` also demonstrates the use of a filter string +to create a GPU device directly. Using a filter string allows much more +fine-grained control for selecting a device. The following example +:ref:`fig-filter-selection` demonstrates usages of device selection using filter +strings. + +.. _fig-filter-selection: + +.. literalinclude:: ../../../../../examples/python/filter_selection.py + :language: python + :lines: 20-21, 23-53 + :caption: Device Creation With Filter Strings + :linenos: + +A possible output for the example :ref:`fig-filter-selection` may be: + +.. program-output:: python ../examples/python/filter_selection.py -r select_using_filter + + +It is also possible to pass a list of devices using a filter string. The +example :ref:`fig-adv-device-selection` demonstrates such a use case. The +filter string ``gpu,cpu`` implies that a GPU should be selected if available, +else a CPU device should be selected. + +.. _fig-adv-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 55-67 + :caption: Selecting a GPU Device if Available + :linenos: + +A possible output for the example :ref:`fig-adv-device-selection` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_gpu_device_if_present + +.. Note:: + A **filter string** is a three-tuple that may specify the *backend*, + *device type*, and *device number* as a colon (:) separated string. The + backend specifies the type of device driver and can have a value such as + *host*, *opencl*, *level-zero*, or *cuda*. The device type can be *host*, + *gpu*, *cpu*, *accelerator*. And, the device number is a numeric value + specifying the ordinality of the device in the listing of devices as + determined by the SYCL runtime. Each of the backend, device type, and device + number value is optional, but at least one of them should be provided, + *i.e.*, ``opencl:gpu:0``, ``gpu:0``, ``gpu``, ``0``, and ``opencl:0`` are + all valid filter strings. + + The device listing including the device number value remain stable for + a given system unless the driver configuration is changed or the SYCL + runtime setting is changed using the ``SYCL_DEVICE_FILTER`` environment + variable. Please refer + :oneapi_filter_selection:`oneAPI filter selection extension <>` for more + detail. + +Advanced Device Selection +------------------------- + +Till now we have discussed device selection using methods that defer the +selection logic to the SYCL runtime. However, real-world applications may +require more precise control over device selection. Dpctl offers a way for users +to accomplish more advanced device selection. + +.. _fig-custom-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 70-91 + :caption: Custom Device Selection + :linenos: + +The example :ref:`fig-custom-device-selection` shows a way of selecting a device +based off a specific hardware property. The :func:`dpctl.get_devices()` returns +a list of all *root* devices on the system, out of that list the devices that +support half-precision floating-point arithmetic are selected. Finally, a +"score" computed using the SYCL runtime's default device scoring logic that is +stored in :attr:`dpctl.SyclDevice.default_selector_score` is used to select a +single device. Refer the documentation of :class:`dpctl.SyclDevice` for a list +of hardware properties that may be used for device selection. + +.. _RootDevice: + +.. Note:: + A **root** device implies an unpartitioned device. A root device can be + partitioned into two or more :ref:`sub-devices ` + based on various criteria. For example, a CPU device with multiple NUMA + domains may be partitioned into multiple sub-devices, each representing a + sub-device. + +A convenience function :func:`dpctl.select_device_with_aspects()` is available +that makes it easy to select a device based on a set of specific aspects. The +example :ref:`fig-select-device-with-aspects` selects a device that +supports double precision arithmetic and SYCL USM shared memory allocation. + +.. _fig-select-device-with-aspects: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 94-103 + :caption: Device Selection Using Aspects + :linenos: + +A possible output for the example :ref:`fig-select-device-with-aspects` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_device_with_aspects diff --git a/docs/docfiles/user_guides/manual/dpctl/platforms.rst b/docs/docfiles/user_guides/manual/dpctl/platforms.rst new file mode 100644 index 0000000000..bf9c0ed981 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/platforms.rst @@ -0,0 +1,35 @@ +.. _querying_platforms: + +######## +Platform +######## + +A platform abstracts a device driver for one or more XPU that is connected to +a host. The :class:`dpctl.SyclPlatform` class represents a platform and +abstracts the :sycl_platform:`sycl::platform <>` SYCL runtime class. + +Listing Available Platforms +--------------------------- + +The platforms available on a system can be queried using the +:func:`dpctl.lsplatform` function. In addition, as illustrated in the following +example it is possible to print out metadata about a platform. + +.. literalinclude:: ../../../../../examples/python/lsplatform.py + :language: python + :lines: 20-41 + :linenos: + +The example can be executed as follows: + +.. code-block:: bash + + python dpctl/examples/python/lsplatform.py -r all + +The possible output for the example may be: + +.. program-output:: python ../examples/python/lsplatform.py -r all + +.. Note:: + The verbosity for the output can be controlled using the ``verbosity`` + keyword argument. Refer :func:`dpctl.lsplatform`. From ddd5c6a5f463dfb7187c3db4a15773b1bc92fdaa Mon Sep 17 00:00:00 2001 From: oleksandr-pavlyk Date: Tue, 21 Dec 2021 01:52:45 -0600 Subject: [PATCH 124/229] Add a page defining the concept of device to dpctl user manual. --- .../manual/dpctl/device_selection.rst | 2 + .../user_guides/manual/dpctl/devices.rst | 140 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 docs/docfiles/user_guides/manual/dpctl/devices.rst diff --git a/docs/docfiles/user_guides/manual/dpctl/device_selection.rst b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst index 75aa7cc18f..b68f930834 100644 --- a/docs/docfiles/user_guides/manual/dpctl/device_selection.rst +++ b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst @@ -37,6 +37,8 @@ A possible output for the example :ref:`fig-gpu-device-selection` may be: .. program-output:: python ../examples/python/device_selection.py -r create_gpu_device +.. _sec-filter-selection: + Selecting a Device Using a Filter String ---------------------------------------- diff --git a/docs/docfiles/user_guides/manual/dpctl/devices.rst b/docs/docfiles/user_guides/manual/dpctl/devices.rst new file mode 100644 index 0000000000..154a772a9e --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/devices.rst @@ -0,0 +1,140 @@ +.. _devices: + +###### +Device +###### + +A device is an abstract representation for an XPU. The :class:`dpctl.SyclDevice` +class represents a device and is a wrapper over the +:sycl_device:`sycl::device <>` SYCL runtime class. + +Creating Devices +---------------- + +We touched upon device creation under the :ref:`device_selection` section. the +:class:`dpctl.SyclDevice` class includes a default constructor to create a +"default" device that is selected by the SYCL runtime. Users can also use +explicit :ref:`filter selector strings ` to create a +device. + +Listing Devices +--------------- + +:py:mod:`dpctl` provides the :func:`dpctl.get_devices` utility function to list +the available devices on a user's system. The list of devices returned depends +on available hardware, installed drivers, as well as by +:dpcpp_envar:`environment variables <>` influencing SYCL runtime +such as ``SYCL_DEVICE_FILTER`` or ``SYCL_DEVICE_ALLOWLIST``. + +.. _fig-listing-devices: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-22, 107-131 + :caption: Listing Available Devices + :linenos: + +A possible output for the example :ref:`fig-listing-devices` may be: + +.. program-output:: python ../examples/python/device_selection.py -r list_devices + +The example :ref:`fig-listing-devices` demonstrates the usage of +:func:`dpctl.get_devices`. The list can be filtered based on +:class:`dpctl.backend` and :class:`dpctl.device_type`. The 0-based ordinal +position of a device in the output of :func:`dpctl.get_devices` corresponds to +the ``device id`` value in the filter selector string corresponding to the +device. For example, ``"opencl:cpu:0"`` refers to the first device in the list +returned by ``dpctl.get_devices(backend="opencl", device_type="cpu")``. If such +a list is empty, device construction call ``dpctl.SyclDevice("opencl:gpu:0")`` +will raise a ``ValueError``. + +.. Note:: + + Unless the system configuration changes, the list of devices returned by + :func:`dpctl.get_devices` and the relative ordering of devices in the list + is stable for every call to the function, even across different runs of an + application. + +Device Aspects and Information Descriptors +------------------------------------------ + +A device can have various *aspects* and *information descriptors* that describe +its hardware characteristics. :sycl_aspects:`Aspects <>` are boolean +characteristics of the device, whereas +:sycl_device_info:`information descriptors <>` are non-boolean characteristics +that provide more verbose information about the device. +:class:`dpctl.SyclDevice` exposes various Python properties that describe a +device's aspects and information descriptors. For example, the property +``has_aspect_fp16`` returns a boolean expression indicating whether a +particular device has aspect ``"fp16"``, indicating whether it supports the +IEEE-754 half-precision floating point type. Whereas, the ``name`` property is +an information descriptor that returns a string with the name of the device. + +.. _fig-available-properties: + +.. code-block:: Python + :caption: Listing Available Device Aspects and Information Descriptors + :linenos: + + import dpctl + import inspect + + def get_properties(cls, prop_name): + "Get name of properties of a class known to have `prop_name`" + known_property_t = type(getattr(cls, prop_name)) + return [n for n, o in inspect.getmembers(cls) if isinstance(o, known_property_t)] + + print(len(get_properties(dpctl.SyclDevice, "name"))) + # Output: 52 + +The example :ref:`fig-available-properties` demonstrates a programmatic way of +listing all the aspects and information descriptor properties in +:class:`dpctl.SyclDevice`. + +.. _sec-devices-sub-devices: + +Sub-devices +----------- + +It is possible for a device to be partitioned into "sub-devices". A sub-device +represents a sub-set of the computational units within a device that are grouped +based on some hardware criteria. For example, a two socket CPU device may be +partitioned into two sub-devices, where each sub-device represents a separate +:numa_domain:`NUMA domain <>`. Depending on the hardware characteristics and +the capabilities of the SYCL runtime, a sub-device may be partitioned further. + +For devices that support partitioning, the +:func:`dpctl.SyclDevice.create_sub_devices` can be used to create a list of +sub-devices. The requested partitioning scheme is indicated with use of the +required ``partition`` keyword. Several types of partitioning schemes are +available: + +* **Equal partitioning** + The partitioning scheme is specified as a list of positive integers + indicating a partitioning with each sub-device having the requested number + of parallel compute units. + +* **Affinity partitioning** + The partitioning scheme is specified as a string indicating an affinity + domain used to create sub-devices that sharing a common resource, such as + certain hardware cache levels. + +.. Note:: + + Use ``partition="next_partitionable"`` to partition along the next level of + architectural hierarchy. + +The following example shows an affinity-based partitioning of a CPU device +into sub-devices based on the available NUMA domains. + +.. _fig-partition-cpu: + +.. literalinclude:: ../../../../../examples/python/subdevices.py + :language: python + :lines: 17, 62-76 + :caption: Partitioning a CPU device + :linenos: + +A possible output for the example :ref:`fig-partition-cpu` may be: + +.. program-output:: python ../examples/python/subdevices.py -r subdivide_by_affinity From cee456fefa06911346b601721163bd68db805a40 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:53:34 -0600 Subject: [PATCH 125/229] Add some new examples references in docs. --- examples/python/_runner.py | 2 +- examples/python/device_selection.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/python/_runner.py b/examples/python/_runner.py index b26865ec5e..552965e3f0 100644 --- a/examples/python/_runner.py +++ b/examples/python/_runner.py @@ -55,7 +55,7 @@ def run_examples(example_description, glbls_dict): print("Available examples:") print(", ".join(fns)) else: - print("No examples are availble.") + print("No examples are available.") exit(0) if args.run == "all": fns = [] diff --git a/examples/python/device_selection.py b/examples/python/device_selection.py index 373b9e401a..2278183c94 100644 --- a/examples/python/device_selection.py +++ b/examples/python/device_selection.py @@ -103,6 +103,32 @@ def create_device_with_aspects(): dev.print_device_info() +def list_devices(): + """Programmatically get a list of the available devices. + + The list can be filtered based on backend or device_type. + """ + print("Get a list of all devices:\n") + + for d in dpctl.get_devices(): + d.print_device_info() + print("=======================================\n") + + print("Get the list of only OpenCL devices:\n") + + for d in dpctl.get_devices(backend="opencl"): + d.print_device_info() + + print("=======================================\n") + + print("Get all OpenCL CPU devices:\n") + + for d in dpctl.get_devices(backend="opencl", device_type="cpu"): + d.print_device_info() + + print("=======================================\n") + + if __name__ == "__main__": import _runner as runner From 5786f74fa1b597a50e12b7b714150755c126e0b8 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:54:03 -0600 Subject: [PATCH 126/229] Add an introduction section to dpctl user manual. --- .../user_guides/manual/dpctl/intro.rst | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/docfiles/user_guides/manual/dpctl/intro.rst diff --git a/docs/docfiles/user_guides/manual/dpctl/intro.rst b/docs/docfiles/user_guides/manual/dpctl/intro.rst new file mode 100644 index 0000000000..327178919e --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/intro.rst @@ -0,0 +1,38 @@ +.. _intro: + +dpctl +----- + +The Data Parallel Control (dpctl) package provides a Python runtime to access a +data-parallel computing resource or *XPU* from another Python application or +library, alleviating the need for the other Python packages to develop such a +runtime themselves. The term XPU denotes a diverse range of compute +architectures such as a CPU, GPU, FPGA, *etc.*, available to programmers on a +modern heterogeneous system. + +The dpctl runtime is built on top of the C++ SYCL standard and is designed to be +both vendor and architecture agnostic. If the underlying SYCL runtime supports +a type of architecture, the dpctl runtime will allow accessing that architecture +from Python. + +In its current form, dpctl relies on certain DPC++ extensions of SYCL standard. +Moreover, the binary distribution of dpctl uses the proprietary Intel(R) oneAPI +DPC++ runtime bundled as part of oneAPI and supports Intel XPU devices only. +However, dpctl is compatible with the runtime of open-source DPC++ SYCL bundle +that can be compiled to support a wide range of architectures including CUDA, +AMD ROC, and HIP. + +The user guide introduces the core features of dpctl and the underlying +concepts. The guide is meant primarily for users of the Python package. Library +and native extension developers should refer to the programmer's guide. + +Table of contents ++++++++++++++++++ + +.. toctree:: + :maxdepth: 2 + + basic_concepts + device_selection + platforms + devices From 23adb091450423f21b13e7ca3667200dd5cb76e2 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:55:42 -0600 Subject: [PATCH 127/229] Add the new dpctl user manual to tocs on other pages. --- docs/docfiles/user_guides/UserManual.rst | 10 ++++++++++ docs/index_doxyrest.rst.in | 7 ++++++- docs/index_no_doxyrest.rst.in | 6 +++++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 docs/docfiles/user_guides/UserManual.rst diff --git a/docs/docfiles/user_guides/UserManual.rst b/docs/docfiles/user_guides/UserManual.rst new file mode 100644 index 0000000000..9b955f1b0c --- /dev/null +++ b/docs/docfiles/user_guides/UserManual.rst @@ -0,0 +1,10 @@ +.. _user_manual: + +########### +User Manual +########### + +.. toctree:: + :maxdepth: 3 + + manual/dpctl/intro diff --git a/docs/index_doxyrest.rst.in b/docs/index_doxyrest.rst.in index cc3c17700b..9e064ed047 100644 --- a/docs/index_doxyrest.rst.in +++ b/docs/index_doxyrest.rst.in @@ -1,10 +1,15 @@ .. include:: ./docfiles/intro.rst + +How-to Guides +============= + .. toctree:: :maxdepth: 1 - :caption: User Guides docfiles/user_guides/QuickStart + docfiles/user_guides/UserManual + .. toctree:: :maxdepth: 1 diff --git a/docs/index_no_doxyrest.rst.in b/docs/index_no_doxyrest.rst.in index 1f367ce2d8..ac57d680ae 100644 --- a/docs/index_no_doxyrest.rst.in +++ b/docs/index_no_doxyrest.rst.in @@ -1,10 +1,14 @@ .. include:: ./docfiles/intro.rst +How-to Guides +============= + .. toctree:: :maxdepth: 1 - :caption: User Guides docfiles/user_guides/QuickStart + docfiles/user_guides/UserManual + .. toctree:: :maxdepth: 1 From 5df1242e43c5b35ee2c3e70fc337ba09e25ff6d4 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:56:22 -0600 Subject: [PATCH 128/229] Fixes and clean ups to exisitng rst pages. --- docs/docfiles/intro.rst | 7 ++-- docs/docfiles/user_guides/QuickStart.rst | 47 +++++++++++------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/docs/docfiles/intro.rst b/docs/docfiles/intro.rst index 115749b3da..892e66af72 100644 --- a/docs/docfiles/intro.rst +++ b/docs/docfiles/intro.rst @@ -2,10 +2,9 @@ Welcome to Data-parallel Control (dpctl)'s documentation! ========================================================= The data-parallel control (dpctl) library provides C and Python bindings for -`SYCL 2020 `_. -The SYCL 2020 features supported by dpctl are limited to those included by -Intel's DPCPP compiler and specifically cover the SYCL runtime classes described -in `Section 4.6 `_ +:sycl_spec_2020:`SYCL 2020 <>`. The SYCL 2020 features supported by dpctl are +limited to those included by Intel's DPCPP compiler and specifically cover the +SYCL runtime classes described in :sycl_runtime_classes:`Section 4.6 <>` of the SYCL 2020 specification. Apart from the bindings for these runtime classes, dpctl includes bindings for SYCL USM memory allocators and deallocators. Dpctl's Python API provides classes that implement diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index dcfede3bba..37d6097e4b 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -4,14 +4,8 @@ Quick Start Guide ################# - -.. contents:: Table of contents - :local: - :backlinks: none - :depth: 3 - Installing from oneAPI ----------------------- +====================== Dpctl is available as part of the oneAPI Intel Distribution of Python (IDP). Please follow `oneAPI installation guide`_ to install oneAPI. In this quick @@ -50,7 +44,7 @@ On Windows `GPU driver installation guide`_. Install Wheel package from Pypi -------------------------------- +=============================== Dpctl can also be istalled from Pypi. @@ -79,7 +73,7 @@ On Windows set PATH=\bin;\Library\bin;%PATH% Building from source --------------------- +==================== To build dpctl from source, we need dpcpp and GPU drivers (and optionally CPU OpenCL drivers). It is preferable to use the dpcpp compiler packaged as part of @@ -87,12 +81,13 @@ oneAPI. However, it is possible to use a custom build of dpcpp to build dpctl, especially if you want to enable CUDA support. Building using oneAPI dpcpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- As before, oneAPI and graphics drivers should be installed on the system prior to proceeding further. -**Activate oneAPI as follows** +Activate oneAPI as follows +~~~~~~~~~~~~~~~~~~~~~~~~~~ On Linux @@ -106,7 +101,8 @@ On Windows call "%ONEAPI_ROOT%\setvars.bat" -**Build and install using conda-build** +Build and install using conda-build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The conda-recipe included with the sources can be used to build the dpctl package. The advantage of this approach is that all dependencies are pulled in @@ -136,7 +132,9 @@ After building the conda package you may install it by executing: You could face issues with conda-build version 3.20. Use conda-build 3.18 instead. -**Build and Install with setuptools** + +Build and install with setuptools +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To build using Python ``setuptools``, the following packages should be installed: @@ -164,13 +162,13 @@ to build and install python setup.py develop Building using custom dpcpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- It is possible to build dpctl from source using .. _DPC++ toolchain: https://github.com/intel/llvm/blob/sycl/sycl/doc/GetStartedGuide.md instead of the DPC++ compiler that comes with oneAPI. One reason for doing this may be to enable support for CUDA devices. -Following steps in :ref:`Build and Install with setuptools` use command line +Following steps in `Build and install with setuptools`_ use command line option :code:`--sycl-compiler-prefix`, for example: .. code-block:: bash @@ -181,7 +179,7 @@ Available options and their descriptions can be retrieved using option :code:`--help`. Using dpctl ------------ +=========== Dpctl requires a DPC++ runtime. When dpctl is installed via conda then it uses the DPC++ runtime from ``dpcpp_cpp_rt`` package that is part of IDP. When using @@ -190,10 +188,10 @@ the system. The easiest way to setup a DPC++ runtime will be by activating oneAPI. Running examples and tests --------------------------- +========================== Running the examples -~~~~~~~~~~~~~~~~~~~~ +-------------------- After setting up dpctl you can try out the Python examples as follows: @@ -213,7 +211,7 @@ located under *examples/cython*. Each example in the folder can be built using examples. Running the Python tests -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The dpctl Python test suite can be executed as follows: @@ -222,14 +220,13 @@ The dpctl Python test suite can be executed as follows: pytest --pyargs dpctl -Building the C API shared library ---------------------------------- +Building the DPCTLSyclInterface library +======================================= -The dpctl C API is a shared library called libDPCTLSyclInterface and is built -together when build the Python package. However, it is possible to only build -the C API as a standalone library. To do so, you will need ``cmake``, +The libDPCTLSyclInterface is a shared library used by the Python package. +To build the library you will need ``DPC++`` toolchain, ``cmake``, ``ninja`` or ``make``, and optionally ``gtest 1.10`` if you wish to run the -C API test suite. +test suite. For example, on Linux the following script can be used to build the C oneAPI library. From db5fd7ac58c7ac1f7303ecea3b375f659fd6356c Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Tue, 21 Dec 2021 01:57:19 -0600 Subject: [PATCH 129/229] Remove urls.rst as it is superseded by urls.json. --- docs/docfiles/urls.rst | 11 ----------- docs/generate_rst.py | 14 -------------- 2 files changed, 25 deletions(-) delete mode 100644 docs/docfiles/urls.rst diff --git a/docs/docfiles/urls.rst b/docs/docfiles/urls.rst deleted file mode 100644 index 25bbf743c8..0000000000 --- a/docs/docfiles/urls.rst +++ /dev/null @@ -1,11 +0,0 @@ - -.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html -.. _Data API: https://data-apis.github.io/array-api/latest/ - -.. _Section 4.6: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes -.. _SYCL 2020 spec: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html -.. _sycl platform: https://sycl.readthedocs.io/en/latest/iface/platform.html -.. _sycl device: https://sycl.readthedocs.io/en/latest/iface/device.html -.. _sycl queue: https://sycl.readthedocs.io/en/latest/iface/queue.html -.. _sycl event: https://sycl.readthedocs.io/en/latest/iface/event.html -.. _sycl context: https://sycl.readthedocs.io/en/latest/iface/context.html diff --git a/docs/generate_rst.py b/docs/generate_rst.py index b2990fef2b..1c80f4bc4a 100644 --- a/docs/generate_rst.py +++ b/docs/generate_rst.py @@ -118,16 +118,6 @@ def _write_underlined(o, s, c): _write_line(o, c * len(s)) -def _write_include_urls(o): - """[summary] - - Args: - o ([type]): [description] - """ - _write_empty_line(o) - _write_line(o, ".. include:: ../urls.rst") - - def _write_hidden_toc(o, list_of_obj_names, prefix_str="", suffix_str=""): """[summary] @@ -368,9 +358,6 @@ def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): output, ".. autofunction:: " + ".".join([cls_qualname, n]), ) - - _write_include_urls(output) - return output.getvalue() @@ -576,7 +563,6 @@ def _write_function_groups_summary(o, mod, groups): _write_empty_line(output) _write_exceptions_summary_table(output, mod) _write_empty_line(output) - _write_include_urls(output) return output.getvalue() From 94d40791d852f6c1579d4cbba48ded65f964555e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 23 Dec 2021 07:50:07 -0600 Subject: [PATCH 130/229] Corrected Equal Partition to Count Partitioning --- docs/docfiles/user_guides/manual/dpctl/devices.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/docfiles/user_guides/manual/dpctl/devices.rst b/docs/docfiles/user_guides/manual/dpctl/devices.rst index 154a772a9e..ca2529b0f4 100644 --- a/docs/docfiles/user_guides/manual/dpctl/devices.rst +++ b/docs/docfiles/user_guides/manual/dpctl/devices.rst @@ -109,10 +109,11 @@ sub-devices. The requested partitioning scheme is indicated with use of the required ``partition`` keyword. Several types of partitioning schemes are available: -* **Equal partitioning** +* **Count partitioning** The partitioning scheme is specified as a list of positive integers indicating a partitioning with each sub-device having the requested number - of parallel compute units. + of parallel compute units, or as a single positive integer indicating + equal-counts partition. * **Affinity partitioning** The partitioning scheme is specified as a string indicating an affinity From 9f729f13a5e028e57d2da65c939e6fd00c5993d5 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 23 Dec 2021 17:29:28 -0600 Subject: [PATCH 131/229] Merge pull request #731 from IntelPython/cpp-style-use-fixed-ubuntu-version cpp_style_check workflow use ubuntu 20.04 --- .github/workflows/cpp_style_checks.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp_style_checks.yml b/.github/workflows/cpp_style_checks.yml index 89ba110ccf..b16530f967 100644 --- a/.github/workflows/cpp_style_checks.yml +++ b/.github/workflows/cpp_style_checks.yml @@ -16,7 +16,12 @@ jobs: steps: - uses: actions/checkout@v2 - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v3.1.0 + uses: jidicula/clang-format-action@v3.5.1 with: clang-format-version: '11' check-path: 'libsyclinterface' + - name: Run clang-format style check for api headers. + uses: jidicula/clang-format-action@v3.5.1 + with: + clang-format-version: '11' + check-path: 'dpctl/apis' From b4b9722a4d2ab35f75844a25b1d46f6669bbabf2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sat, 25 Dec 2021 08:12:15 -0600 Subject: [PATCH 132/229] Use updated OCL drivers: 12.13.11.0.23_rel, and TBB 2021.5.0 (#732) --- .github/workflows/os-llvm-sycl-build.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index 92b85e1318..47b183cd7e 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -10,9 +10,12 @@ jobs: runs-on: ubuntu-20.04 env: - OCLCPUEXP_FN: oclcpuexp-2021.12.9.0.24_rel.tar.gz - FPGAEMU_FN: fpgaemu-2021.12.9.0.24_rel.tar.gz - TBB_FN: oneapi-tbb-2021.4.0-lin.tgz + DOWNLOAD_URL_PREFIX: https://github.com/intel/llvm/releases/download + DRIVER_PATH: 2021-WW50 + OCLCPUEXP_FN: oclcpuexp-2021.13.11.0.23_rel.tar.gz + FPGAEMU_FN: fpgaemu-2021.13.11.0.23_rel.tar.gz + TBB_URL: https://github.com/oneapi-src/oneTBB/releases/download/v2021.5.0 + TBB_FN: oneapi-tbb-2021.5.0-lin.tgz steps: - name: Cancel Previous Runs @@ -45,19 +48,18 @@ jobs: if [[ -f bundle_id.txt && ( "$(cat bundle_id.txt)" == "${LATEST_LLVM_TAG_SHA}" ) ]]; then echo "Using cached download of ${LATEST_LLVM_TAG}" else - export DOWNLOAD_URL_PREFIX=https://github.com/intel/llvm/releases/download rm -rf dpcpp-compiler.tar.gz wget ${DOWNLOAD_URL_PREFIX}/${NIGHTLY_TAG}/dpcpp-compiler.tar.gz && echo ${LATEST_LLVM_TAG_SHA} > bundle_id.txt || rm -rf bundle_id.txt - [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-09/${OCLCPUEXP_FN} || rm -rf bundle_id.txt - [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-09/${FPGAEMU_FN} || rm -rf bundle_id.txt - [ -f ${TBB_FN} ] || wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.4.0/${TBB_FN} || rm -rf bundle_id.txt + [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/${DRIVER_PATH}/${OCLCPUEXP_FN} || rm -rf bundle_id.txt + [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/${DRIVER_PATH}/${FPGAEMU_FN} || rm -rf bundle_id.txt + [ -f ${TBB_FN} ] || wget ${TBB_URL}/${TBB_FN} || rm -rf bundle_id.txt rm -rf dpcpp_compiler tar xf dpcpp-compiler.tar.gz mkdir -p oclcpuexp mkdir -p fpgaemu [ -d oclcpuexp/x64 ] || tar xf ${OCLCPUEXP_FN} -C oclcpuexp [ -d fpgaemu/x64 ] || tar xf ${FPGAEMU_FN} -C fpgaemu - [ -d oneapi-tbb-2021.4.0/lib ] || tar xf ${TBB_FN} + [ -d oneapi-tbb-2021.5.0/lib ] || tar xf ${TBB_FN} mkdir -p dpcpp_compiler/lib mkdir -p dpcpp_compiler/lib/oclfpga touch dpcpp_compiler/lib/oclfpga/fpgavars.sh @@ -91,7 +93,7 @@ jobs: source ${SYCL_BUNDLE_FOLDER}/dpcpp_compiler/startup.sh export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oclcpuexp/x64:${LD_LIBRARY_PATH} export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/fpgaemu/x64:${LD_LIBRARY_PATH} - export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oneapi-tbb-2021.4.0/lib/intel64/gcc4.8:${LD_LIBRARY_PATH} + export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oneapi-tbb-2021.5.0/lib/intel64/gcc4.8:${LD_LIBRARY_PATH} export OCL_ICD_FILENAMES=libintelocl.so:libintelocl_emu.so clang++ --version sycl-ls From 23b8884ba361d8051af6d7d4636ab4b664dea9c2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 28 Dec 2021 12:27:46 -0600 Subject: [PATCH 133/229] Synchronization function release GIL DPCTLQueue_Wait and friends are declared nogil, and their use is moved into `with nogil` context. This allows us to use host_task to decrement reference count on a Python object without introducing a deadlock. --- dpctl/_backend.pxd | 6 +++--- dpctl/_sycl_event.pyx | 9 ++++++--- dpctl/_sycl_queue.pyx | 12 ++++++++---- dpctl/memory/_memory.pyx | 15 ++++++++++----- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index ab4af7d136..0ba086db79 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -227,8 +227,8 @@ cdef extern from "syclinterface/dpctl_sycl_device_selector_interface.h": cdef extern from "syclinterface/dpctl_sycl_event_interface.h": cdef DPCTLSyclEventRef DPCTLEvent_Create() cdef DPCTLSyclEventRef DPCTLEvent_Copy(const DPCTLSyclEventRef ERef) - cdef void DPCTLEvent_Wait(DPCTLSyclEventRef ERef) - cdef void DPCTLEvent_WaitAndThrow(DPCTLSyclEventRef ERef) + cdef void DPCTLEvent_Wait(DPCTLSyclEventRef ERef) nogil + cdef void DPCTLEvent_WaitAndThrow(DPCTLSyclEventRef ERef) nogil cdef void DPCTLEvent_Delete(DPCTLSyclEventRef ERef) cdef _event_status_type DPCTLEvent_GetCommandExecutionStatus(DPCTLSyclEventRef ERef) cdef _backend_type DPCTLEvent_GetBackend(DPCTLSyclEventRef ERef) @@ -356,7 +356,7 @@ cdef extern from "syclinterface/dpctl_sycl_queue_interface.h": size_t NDims, const DPCTLSyclEventRef *DepEvents, size_t NDepEvents) - cdef void DPCTLQueue_Wait(const DPCTLSyclQueueRef QRef) + cdef void DPCTLQueue_Wait(const DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclEventRef DPCTLQueue_Memcpy( const DPCTLSyclQueueRef Q, void *Dest, diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index 5dfb8ee10b..1a9f089928 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -98,7 +98,8 @@ cdef class _SyclEvent: def __dealloc__(self): if (self._event_ref): - DPCTLEvent_Wait(self._event_ref) + with nogil: + DPCTLEvent_Wait(self._event_ref) DPCTLEvent_Delete(self._event_ref) self._event_ref = NULL self.args = None @@ -224,7 +225,8 @@ cdef class SyclEvent(_SyclEvent): @staticmethod cdef void _wait(SyclEvent event): - DPCTLEvent_WaitAndThrow(event._event_ref) + with nogil: + DPCTLEvent_WaitAndThrow(event._event_ref) @staticmethod def wait_for(event): @@ -370,4 +372,5 @@ cdef class SyclEvent(_SyclEvent): cpdef void wait(self): "Synchronously wait for completion of this event." - DPCTLEvent_Wait(self._event_ref) + with nogil: + DPCTLEvent_Wait(self._event_ref) diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 614b7b2ae8..2f8aea5515 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -824,7 +824,8 @@ cdef class SyclQueue(_SyclQueue): return SyclEvent._create(Eref, args) cpdef void wait(self): - DPCTLQueue_Wait(self._queue_ref) + with nogil: + DPCTLQueue_Wait(self._queue_ref) cpdef memcpy(self, dest, src, size_t count): cdef void *c_dest @@ -846,7 +847,8 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.memcpy operation encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef prefetch(self, mem, size_t count=0): @@ -866,7 +868,8 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.prefetch encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef mem_advise(self, mem, size_t count, int advice): @@ -886,7 +889,8 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.mem_advise operation encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) @property diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 93f69ce548..a6a1c63f49 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -94,7 +94,8 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, src_ptr, nbytes ) - DPCTLEvent_Wait(E1Ref) + with nogil: + DPCTLEvent_Wait(E1Ref) E2Ref = DPCTLQueue_Memcpy( dest_queue.get_queue_ref(), @@ -102,7 +103,8 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, &host_buf[0], nbytes ) - DPCTLEvent_Wait(E2Ref) + with nogil: + DPCTLEvent_Wait(E2Ref) DPCTLEvent_Delete(E1Ref) DPCTLEvent_Delete(E2Ref) @@ -398,7 +400,8 @@ cdef class _Memory: self.memory_ptr, # source self.nbytes ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) return obj @@ -423,7 +426,8 @@ cdef class _Memory: &host_buf[0], # source buf_len ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef copy_from_device(self, object sycl_usm_ary): @@ -465,7 +469,8 @@ cdef class _Memory: src_buf.p, src_buf.nbytes ) - DPCTLEvent_Wait(ERef) + with nogil: + DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) else: copy_via_host( From cd2772835a6694c1ad50325f8c0e7b35d2a1f406 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 5 Jan 2022 11:44:20 -0600 Subject: [PATCH 134/229] Used 1-line context manage syntax to avoid coverage drop --- dpctl/_sycl_event.pyx | 9 +++------ dpctl/_sycl_queue.pyx | 12 ++++-------- dpctl/memory/_memory.pyx | 15 +++++---------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index 1a9f089928..12f7c376b2 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -98,8 +98,7 @@ cdef class _SyclEvent: def __dealloc__(self): if (self._event_ref): - with nogil: - DPCTLEvent_Wait(self._event_ref) + with nogil: DPCTLEvent_Wait(self._event_ref) DPCTLEvent_Delete(self._event_ref) self._event_ref = NULL self.args = None @@ -225,8 +224,7 @@ cdef class SyclEvent(_SyclEvent): @staticmethod cdef void _wait(SyclEvent event): - with nogil: - DPCTLEvent_WaitAndThrow(event._event_ref) + with nogil: DPCTLEvent_WaitAndThrow(event._event_ref) @staticmethod def wait_for(event): @@ -372,5 +370,4 @@ cdef class SyclEvent(_SyclEvent): cpdef void wait(self): "Synchronously wait for completion of this event." - with nogil: - DPCTLEvent_Wait(self._event_ref) + with nogil: DPCTLEvent_Wait(self._event_ref) diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 2f8aea5515..05bd2f753b 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -824,8 +824,7 @@ cdef class SyclQueue(_SyclQueue): return SyclEvent._create(Eref, args) cpdef void wait(self): - with nogil: - DPCTLQueue_Wait(self._queue_ref) + with nogil: DPCTLQueue_Wait(self._queue_ref) cpdef memcpy(self, dest, src, size_t count): cdef void *c_dest @@ -847,8 +846,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.memcpy operation encountered an error" ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef prefetch(self, mem, size_t count=0): @@ -868,8 +866,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.prefetch encountered an error" ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef mem_advise(self, mem, size_t count, int advice): @@ -889,8 +886,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.mem_advise operation encountered an error" ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) @property diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index a6a1c63f49..76d6255df3 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -94,8 +94,7 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, src_ptr, nbytes ) - with nogil: - DPCTLEvent_Wait(E1Ref) + with nogil: DPCTLEvent_Wait(E1Ref) E2Ref = DPCTLQueue_Memcpy( dest_queue.get_queue_ref(), @@ -103,8 +102,7 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, &host_buf[0], nbytes ) - with nogil: - DPCTLEvent_Wait(E2Ref) + with nogil: DPCTLEvent_Wait(E2Ref) DPCTLEvent_Delete(E1Ref) DPCTLEvent_Delete(E2Ref) @@ -400,8 +398,7 @@ cdef class _Memory: self.memory_ptr, # source self.nbytes ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) return obj @@ -426,8 +423,7 @@ cdef class _Memory: &host_buf[0], # source buf_len ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef copy_from_device(self, object sycl_usm_ary): @@ -469,8 +465,7 @@ cdef class _Memory: src_buf.p, src_buf.nbytes ) - with nogil: - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) else: copy_via_host( From 0ae7844fea8991b3175a655e876cad814cb434c0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 5 Jan 2022 12:50:42 -0600 Subject: [PATCH 135/229] USM alloc. fns declared nogil, used inside `with nogil` context Large USM allocations can be constly, so releasing GIL while invoking them may be a good idea. --- dpctl/_backend.pxd | 14 ++++++++------ dpctl/memory/_memory.pyx | 20 +++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index 0ba086db79..b4788ab493 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -394,23 +394,25 @@ cdef extern from "syclinterface/dpctl_sycl_queue_manager.h": cdef extern from "syclinterface/dpctl_sycl_usm_interface.h": cdef DPCTLSyclUSMRef DPCTLmalloc_shared( size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLmalloc_host( size_t size, - DPCTLSyclQueueRef QRef) - cdef DPCTLSyclUSMRef DPCTLmalloc_device(size_t size, DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil + cdef DPCTLSyclUSMRef DPCTLmalloc_device( + size_t size, + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_shared( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_host( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_device( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef void DPCTLfree_with_queue( DPCTLSyclUSMRef MRef, DPCTLSyclQueueRef QRef) diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 76d6255df3..741d72e22d 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -142,6 +142,7 @@ cdef class _Memory: cdef _cinit_alloc(self, Py_ssize_t alignment, Py_ssize_t nbytes, bytes ptr_type, SyclQueue queue): cdef DPCTLSyclUSMRef p = NULL + cdef DPCTLSyclQueueRef QRef = NULL self._cinit_empty() @@ -149,27 +150,28 @@ cdef class _Memory: if queue is None: queue = dpctl.SyclQueue() + QRef = queue.get_queue_ref() if (ptr_type == b"shared"): if alignment > 0: - p = DPCTLaligned_alloc_shared( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_shared( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_shared(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_shared(nbytes, QRef) elif (ptr_type == b"host"): if alignment > 0: - p = DPCTLaligned_alloc_host( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_host( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_host(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_host(nbytes, QRef) elif (ptr_type == b"device"): if (alignment > 0): - p = DPCTLaligned_alloc_device( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_device( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_device(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_device(nbytes, QRef) else: raise RuntimeError( "Pointer type is unknown: {}".format( From 6b81ed0eb2748f9076c629253ea770c97c3709f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 5 Jan 2022 14:53:19 -0600 Subject: [PATCH 136/229] Updated setup's classifiers keyword argument (#735) Remove declaration for support of Python 3.6, added declaration of support of Python 3.9 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b624ae473a..d88580e242 100644 --- a/setup.py +++ b/setup.py @@ -492,8 +492,8 @@ def _get_cmdclass(): keywords="dpctl", classifiers=[ "Development Status :: 3 - Alpha", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], ) From 43f33cca3c465a82b4bf1d2328f9c9fca6f6f5a5 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 5 Jan 2022 17:12:22 -0600 Subject: [PATCH 137/229] Merge pull request #737 from IntelPython/fix-get-num-devices-crash DPCTLDeviceMgr_GetNumDevices should not operate on rejected devices --- libsyclinterface/source/dpctl_sycl_device_manager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libsyclinterface/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp index 06d90c039c..00b81395a9 100644 --- a/libsyclinterface/source/dpctl_sycl_device_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_manager.cpp @@ -286,7 +286,10 @@ size_t DPCTLDeviceMgr_GetNumDevices(int device_identifier) if (!device_identifier) return 0; + default_selector mRanker; for (const auto &entry : cache) { + if (mRanker(entry.first) < 0) + continue; auto Bty(DPCTL_SyclBackendToDPCTLBackendType( entry.first.get_platform().get_backend())); auto Dty(DPCTL_SyclDeviceTypeToDPCTLDeviceType( From f3160da24f5fa0427b0f4ffd9154aaab56a3ae00 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 7 Jan 2022 10:03:25 -0600 Subject: [PATCH 138/229] Added smoke step for testing_${{ platform }} jobs (#733) --- .github/workflows/conda-package.yml | 45 ++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index f4c15abfb2..dff09e302f 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -152,6 +152,11 @@ jobs: conda install $PACKAGE_NAME pytest python=${{ matrix.python }} $CHANNELS # Test installed packages conda list + - name: Smoke test + run: | + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + python -c "import dpctl; dpctl.lsplatform()" - name: Run tests run: | # echo "libintelocl.so" | tee /etc/OpenCL/vendors/intel-cpu.icd @@ -213,10 +218,48 @@ jobs: # Test installed packages conda list - name: Add library - run: echo "OCL_ICD_FILENAMES=C:\Miniconda\Library\lib\intelocl64.dll" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + run: | + echo "OCL_ICD_FILENAMES=C:\Miniconda\Library\lib\intelocl64.dll" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + try {$list = Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors | Select-Object -ExpandProperty Property } catch {$list=@()} + if ($list.count -eq 0) { + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos)) { + New-Item -Path HKLM:\SOFTWARE\Khronos + } + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos\OpenCL)) { + New-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL + } + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors)) { + New-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors + } + New-ItemProperty -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors -Name C:\Miniconda\Library\lib\intelocl64.dll -Value 0 + try {$list = Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors | Select-Object -ExpandProperty Property } catch {$list=@()} + Write-Output $(Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors) + # Now copy OpenCL.dll into system folder + $system_ocl_icd_loader="C:\Windows\System32\OpenCL.dll" + $python_ocl_icd_loader="C:\Miniconda\Library\bin\OpenCL.dll" + Copy-Item -Path $python_ocl_icd_loader -Destination $system_ocl_icd_loader + if (Test-Path -Path $system_ocl_icd_loader) { + Write-Output "$system_ocl_icd_loader has been copied" + $acl = Get-Acl $system_ocl_icd_loader + Write-Output $acl + } else { + Write-Output "OCL-ICD-Loader was not copied" + } + # Create symbolic links to tbb next to OpenCL's task_executor, where it expects to find them + New-Item -ItemType HardLink -Path "C:\Miniconda\Library\lib\tbb12.dll" -Target "C:\Miniconda\Library\bin\tbb12.dll" + New-Item -ItemType HardLink -Path "C:\Miniconda\Library\lib\tbbmalloc.dll" -Target "C:\Miniconda\Library\bin\tbbmalloc.dll" + } + - name: Smoke test + run: | + set SYCL_ENABLE_HOST_DEVICE=1 + & { [Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Miniconda\Library\bin\", [EnvironmentVariableTarget]::Machine) } + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpctl; print(dpctl.get_devices(backend='opencl', device_type='gpu'))" + python -c "import dpctl; print(dpctl.get_num_devices(backend='opencl', device_type='gpu'))" - name: Run tests run: | set SYCL_ENABLE_HOST_DEVICE=1 + & { [Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Miniconda\Library\bin\", [EnvironmentVariableTarget]::Machine) } python -m pytest --pyargs ${{ env.MODULE_NAME }} upload_linux: From 4e1f63d7b3d4a73d3f1f27fd0ec432e9b0e94dc4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 7 Jan 2022 11:04:03 -0600 Subject: [PATCH 139/229] DeviceCache builder uses try/catch (#738) If exception ocurred, error handler is invoked, and the device->context map is not cached. std::terminate call was removed (dead code). --- .../source/dpctl_sycl_device_manager.cpp | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp index 00b81395a9..bb140dbc0c 100644 --- a/libsyclinterface/source/dpctl_sycl_device_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_manager.cpp @@ -126,19 +126,18 @@ struct DeviceCacheBuilder if (mRanker(D) < 0) continue; - // Per https://github.com/intel/llvm/blob/sycl/sycl/doc/ - // extensions/PlatformContext/PlatformContext.adoc - // sycl::queue(D) would create default platform context - // for capable compiler, sycl::context(D) otherwise - auto Q = queue(D); - auto Ctx = Q.get_context(); - auto entry = cache_l.emplace(D, Ctx); - - if (!entry.second) { - error_handler("Fatal Error during device cache " - "construction.", - __FILE__, __func__, __LINE__); - std::terminate(); + try { + // Per https://github.com/intel/llvm/blob/sycl/sycl/doc/ + // extensions/PlatformContext/PlatformContext.adoc + // sycl::queue(D) would create default platform context + // for capable compiler, sycl::context(D) otherwise + auto Q = queue(D); + auto Ctx = Q.get_context(); + cache_l.emplace(D, Ctx); + } catch (const std::exception &e) { + // Nothing is added to the cache_l by guarantees of + // emplace + error_handler(e, __FILE__, __func__, __LINE__); } } } From 0e595728eb9dfc943774b654035e9b339bde8dce Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 7 Jan 2022 12:16:33 -0600 Subject: [PATCH 140/229] Set TBB_DLL_PATH rather than create hard links in the environment (#739) --- .github/workflows/conda-package.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index dff09e302f..9e07dc8f69 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -245,9 +245,8 @@ jobs: } else { Write-Output "OCL-ICD-Loader was not copied" } - # Create symbolic links to tbb next to OpenCL's task_executor, where it expects to find them - New-Item -ItemType HardLink -Path "C:\Miniconda\Library\lib\tbb12.dll" -Target "C:\Miniconda\Library\bin\tbb12.dll" - New-Item -ItemType HardLink -Path "C:\Miniconda\Library\lib\tbbmalloc.dll" -Target "C:\Miniconda\Library\bin\tbbmalloc.dll" + # Variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default + echo "TBB_DLL_PATH=C:\Miniconda\Library\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append } - name: Smoke test run: | From 53fe358c6a47b0af8b77af838f08e6b2e696e0e3 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 14 Dec 2021 18:46:29 -0600 Subject: [PATCH 141/229] Added more public API methods for _Memory object --- dpctl/memory/_memory.pyx | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 741d72e22d..46cb63f826 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -752,10 +752,28 @@ def as_usm_memory(obj): cdef api DPCTLSyclUSMRef get_usm_pointer(_Memory obj): + "Pointer of USM allocation" return obj.memory_ptr -cdef api DPCTLSyclContextRef get_context(_Memory obj): +cdef api DPCTLSyclContextRef get_context_ref(_Memory obj): + "Context reference to which USM allocation is bound" return obj.queue._context.get_context_ref() +cdef api DPCTLSyclQueueRef get_queue_ref(_Memory obj): + """Queue associated with this allocation, used + for copying, population, etc.""" + return obj.queue.get_queue_ref() + cdef api size_t get_nbytes(_Memory obj): + "Size of the allocation in bytes." return obj.nbytes + +cdef api object make_Memory( + DPCTLSyclUSMRef ptr, + size_t nbytes, + DPCTLSyclQueueRef QRef, + object owner +): + return _Memory.create_from_usm_pointer_size_qref( + ptr, nbytes, QRef, memory_owner=owner + ) From 76186347d891268655293a0b61b5e141152d2979 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 15 Dec 2021 11:43:02 -0600 Subject: [PATCH 142/229] changed test to reflect change in public API function name change --- dpctl/tests/test_sycl_usm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index 0bb68edaa6..cff646a39d 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -531,7 +531,7 @@ def test_cpython_api(memory_ctor): mod = sys.modules[mobj.__class__.__module__] # get capsules storing function pointers mem_ptr_fn_cap = mod.__pyx_capi__["get_usm_pointer"] - mem_ctx_fn_cap = mod.__pyx_capi__["get_context"] + mem_ctx_ref_fn_cap = mod.__pyx_capi__["get_context_ref"] mem_nby_fn_cap = mod.__pyx_capi__["get_nbytes"] # construct Python callable to invoke "get_usm_pointer" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer @@ -540,21 +540,21 @@ def test_cpython_api(memory_ctor): mem_ptr_fn_ptr = cap_ptr_fn( mem_ptr_fn_cap, b"DPCTLSyclUSMRef (struct Py_MemoryObject *)" ) - mem_ctx_fn_ptr = cap_ptr_fn( - mem_ctx_fn_cap, b"DPCTLSyclContextRef (struct Py_MemoryObject *)" + mem_ctx_ref_fn_ptr = cap_ptr_fn( + mem_ctx_ref_fn_cap, b"DPCTLSyclContextRef (struct Py_MemoryObject *)" ) mem_nby_fn_ptr = cap_ptr_fn( mem_nby_fn_cap, b"size_t (struct Py_MemoryObject *)" ) callable_maker = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object) get_ptr_fn = callable_maker(mem_ptr_fn_ptr) - get_ctx_fn = callable_maker(mem_ctx_fn_ptr) + get_ctx_ref_fn = callable_maker(mem_ctx_ref_fn_ptr) get_nby_fn = callable_maker(mem_nby_fn_ptr) capi_ptr = get_ptr_fn(mobj) direct_ptr = mobj._pointer assert capi_ptr == direct_ptr - capi_ctx_ref = get_ctx_fn(mobj) + capi_ctx_ref = get_ctx_ref_fn(mobj) direct_ctx_ref = mobj._context.addressof_ref() assert capi_ctx_ref == direct_ctx_ref capi_nbytes = get_nby_fn(mobj) From 63109d74d84308b3b69b2312c5fa7328c5a15eb1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 16 Dec 2021 16:14:45 -0600 Subject: [PATCH 143/229] Added Memory C-API functions to dpctl_capi.h Renamed functions to avoid name clashes for get_queue_ref/get_context_ref defined for different signatures. (C-API requires names to be different). Adjsuted tests to reflect changes in C-API names --- dpctl/apis/include/dpctl_capi.h | 4 +++- dpctl/memory/_memory.pyx | 8 ++++---- dpctl/tests/test_sycl_usm.py | 14 +++++++++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/dpctl/apis/include/dpctl_capi.h b/dpctl/apis/include/dpctl_capi.h index 0a484bcff7..2be253a618 100644 --- a/dpctl/apis/include/dpctl_capi.h +++ b/dpctl/apis/include/dpctl_capi.h @@ -36,6 +36,8 @@ #include "../_sycl_event_api.h" #include "../_sycl_queue.h" #include "../_sycl_queue_api.h" +#include "../memory/_memory.h" +#include "../memory/_memory_api.h" // clang-format on /* @@ -50,6 +52,6 @@ void import_dpctl(void) import_dpctl___sycl_context(); import_dpctl___sycl_event(); import_dpctl___sycl_queue(); - + import_dpctl__memory___memory(); return; } diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 46cb63f826..46be4c1c59 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -751,20 +751,20 @@ def as_usm_memory(obj): ) -cdef api DPCTLSyclUSMRef get_usm_pointer(_Memory obj): +cdef api DPCTLSyclUSMRef Memory_get_usm_pointer(_Memory obj): "Pointer of USM allocation" return obj.memory_ptr -cdef api DPCTLSyclContextRef get_context_ref(_Memory obj): +cdef api DPCTLSyclContextRef Memory_get_context_ref(_Memory obj): "Context reference to which USM allocation is bound" return obj.queue._context.get_context_ref() -cdef api DPCTLSyclQueueRef get_queue_ref(_Memory obj): +cdef api DPCTLSyclQueueRef Memory_get_queue_ref(_Memory obj): """Queue associated with this allocation, used for copying, population, etc.""" return obj.queue.get_queue_ref() -cdef api size_t get_nbytes(_Memory obj): +cdef api size_t Memory_get_nbytes(_Memory obj): "Size of the allocation in bytes." return obj.nbytes diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index cff646a39d..8ed932fa9d 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -530,9 +530,10 @@ def test_cpython_api(memory_ctor): mobj = memory_ctor(1024) mod = sys.modules[mobj.__class__.__module__] # get capsules storing function pointers - mem_ptr_fn_cap = mod.__pyx_capi__["get_usm_pointer"] - mem_ctx_ref_fn_cap = mod.__pyx_capi__["get_context_ref"] - mem_nby_fn_cap = mod.__pyx_capi__["get_nbytes"] + mem_ptr_fn_cap = mod.__pyx_capi__["Memory_get_usm_pointer"] + mem_q_ref_fn_cap = mod.__pyx_capi__["Memory_get_queue_ref"] + mem_ctx_ref_fn_cap = mod.__pyx_capi__["Memory_get_context_ref"] + mem_nby_fn_cap = mod.__pyx_capi__["Memory_get_nbytes"] # construct Python callable to invoke "get_usm_pointer" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p @@ -543,12 +544,16 @@ def test_cpython_api(memory_ctor): mem_ctx_ref_fn_ptr = cap_ptr_fn( mem_ctx_ref_fn_cap, b"DPCTLSyclContextRef (struct Py_MemoryObject *)" ) + mem_q_ref_fn_ptr = cap_ptr_fn( + mem_q_ref_fn_cap, b"DPCTLSyclQueueRef (struct Py_MemoryObject *)" + ) mem_nby_fn_ptr = cap_ptr_fn( mem_nby_fn_cap, b"size_t (struct Py_MemoryObject *)" ) callable_maker = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object) get_ptr_fn = callable_maker(mem_ptr_fn_ptr) get_ctx_ref_fn = callable_maker(mem_ctx_ref_fn_ptr) + get_q_ref_fn = callable_maker(mem_q_ref_fn_ptr) get_nby_fn = callable_maker(mem_nby_fn_ptr) capi_ptr = get_ptr_fn(mobj) @@ -557,6 +562,9 @@ def test_cpython_api(memory_ctor): capi_ctx_ref = get_ctx_ref_fn(mobj) direct_ctx_ref = mobj._context.addressof_ref() assert capi_ctx_ref == direct_ctx_ref + capi_q_ref = get_q_ref_fn(mobj) + direct_q_ref = mobj.sycl_queue.addressof_ref() + assert capi_q_ref == direct_q_ref capi_nbytes = get_nby_fn(mobj) direct_nbytes = mobj.nbytes assert capi_nbytes == direct_nbytes From c771d1a1640e2534a262863e338978d25cb29c4c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 7 Jan 2022 15:42:35 -0600 Subject: [PATCH 144/229] Renamed Python C-API functions to ObjectName_CamelCasedFunctionName --- dpctl/_sycl_context.pyx | 4 ++-- dpctl/_sycl_device.pyx | 4 ++-- dpctl/_sycl_event.pyx | 4 ++-- dpctl/_sycl_queue.pyx | 4 ++-- dpctl/apis/include/dpctl4pybind11.hpp | 22 +++++++++++----------- dpctl/memory/_memory.pyx | 10 +++++----- dpctl/tests/test_sycl_context.py | 16 ++++++++-------- dpctl/tests/test_sycl_device.py | 16 ++++++++-------- dpctl/tests/test_sycl_event.py | 16 ++++++++-------- dpctl/tests/test_sycl_queue.py | 16 ++++++++-------- dpctl/tests/test_sycl_usm.py | 10 +++++----- 11 files changed, 61 insertions(+), 61 deletions(-) diff --git a/dpctl/_sycl_context.pyx b/dpctl/_sycl_context.pyx index 87103d7c8a..247a5d0f68 100644 --- a/dpctl/_sycl_context.pyx +++ b/dpctl/_sycl_context.pyx @@ -480,7 +480,7 @@ cdef class SyclContext(_SyclContext): &_context_capsule_deleter ) -cdef api DPCTLSyclContextRef get_context_ref(SyclContext ctx): +cdef api DPCTLSyclContextRef SyclContext_GetContextRef(SyclContext ctx): """ C-API function to get opaque context reference from :class:`dpctl.SyclContext` instance. @@ -488,7 +488,7 @@ cdef api DPCTLSyclContextRef get_context_ref(SyclContext ctx): return ctx.get_context_ref() -cdef api SyclContext make_SyclContext(DPCTLSyclContextRef CRef): +cdef api SyclContext SyclContext_Make(DPCTLSyclContextRef CRef): """ C-API function to create :class:`dpctl.SyclContext` instance from the given opaque context reference. diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 83b409c46c..2fc00763b0 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -1130,7 +1130,7 @@ cdef class SyclDevice(_SyclDevice): else: return str(relId) -cdef api DPCTLSyclDeviceRef get_device_ref(SyclDevice dev): +cdef api DPCTLSyclDeviceRef SyclDevice_GetDeviceRef(SyclDevice dev): """ C-API function to get opaque device reference from :class:`dpctl.SyclDevice` instance. @@ -1138,7 +1138,7 @@ cdef api DPCTLSyclDeviceRef get_device_ref(SyclDevice dev): return dev.get_device_ref() -cdef api SyclDevice make_SyclDevice(DPCTLSyclDeviceRef DRef): +cdef api SyclDevice SyclDevice_Make(DPCTLSyclDeviceRef DRef): """ C-API function to create :class:`dpctl.SyclDevice` instance from the given opaque device reference. diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index 12f7c376b2..bd680eadf7 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -57,14 +57,14 @@ __all__ = [ _logger = logging.getLogger(__name__) -cdef api DPCTLSyclEventRef get_event_ref(SyclEvent ev): +cdef api DPCTLSyclEventRef SyclEvent_GetEventRef(SyclEvent ev): """ C-API function to access opaque event reference from Python object of type :class:`dpctl.SyclEvent`. """ return ev.get_event_ref() -cdef api SyclEvent make_SyclEvent(DPCTLSyclEventRef ERef): +cdef api SyclEvent SyclEvent_Make(DPCTLSyclEventRef ERef): """ C-API function to create :class:`dpctl.SyclEvent` instance from opaque sycl event reference. diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 05bd2f753b..1e1b15813b 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -1001,7 +1001,7 @@ cdef class SyclQueue(_SyclQueue): self.sycl_device.print_device_info() -cdef api DPCTLSyclQueueRef get_queue_ref(SyclQueue q): +cdef api DPCTLSyclQueueRef SyclQueue_GetQueueRef(SyclQueue q): """ C-API function to get opaque queue reference from :class:`dpctl.SyclQueue` instance. @@ -1009,7 +1009,7 @@ cdef api DPCTLSyclQueueRef get_queue_ref(SyclQueue q): return q.get_queue_ref() -cdef api SyclQueue make_SyclQueue(DPCTLSyclQueueRef QRef): +cdef api SyclQueue SyclQueue_Make(DPCTLSyclQueueRef QRef): """ C-API function to create :class:`dpctl.SyclQueue` instance from the given opaque queue reference. diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp index 4182a2d9bb..b2e5e05633 100644 --- a/dpctl/apis/include/dpctl4pybind11.hpp +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -49,8 +49,8 @@ template <> struct type_caster { PyObject *source = src.ptr(); if (PyObject_TypeCheck(source, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(source)); + DPCTLSyclQueueRef QRef = SyclQueue_GetQueueRef( + reinterpret_cast(source)); sycl::queue *q = reinterpret_cast(QRef); value = *q; return true; @@ -63,7 +63,7 @@ template <> struct type_caster static handle cast(sycl::queue src, return_value_policy, handle) { - auto tmp = make_SyclQueue(reinterpret_cast(&src)); + auto tmp = SyclQueue_Make(reinterpret_cast(&src)); return handle(reinterpret_cast(tmp)); } }; @@ -87,8 +87,8 @@ template <> struct type_caster { PyObject *source = src.ptr(); if (PyObject_TypeCheck(source, &PySyclDeviceType)) { - DPCTLSyclDeviceRef DRef = - get_device_ref(reinterpret_cast(source)); + DPCTLSyclDeviceRef DRef = SyclDevice_GetDeviceRef( + reinterpret_cast(source)); sycl::device *d = reinterpret_cast(DRef); value = *d; return true; @@ -101,7 +101,7 @@ template <> struct type_caster static handle cast(sycl::device src, return_value_policy, handle) { - auto tmp = make_SyclDevice(reinterpret_cast(&src)); + auto tmp = SyclDevice_Make(reinterpret_cast(&src)); return handle(reinterpret_cast(tmp)); } }; @@ -125,7 +125,7 @@ template <> struct type_caster { PyObject *source = src.ptr(); if (PyObject_TypeCheck(source, &PySyclContextType)) { - DPCTLSyclContextRef CRef = get_context_ref( + DPCTLSyclContextRef CRef = SyclContext_GetContextRef( reinterpret_cast(source)); sycl::context *ctx = reinterpret_cast(CRef); value = *ctx; @@ -140,7 +140,7 @@ template <> struct type_caster static handle cast(sycl::context src, return_value_policy, handle) { auto tmp = - make_SyclContext(reinterpret_cast(&src)); + SyclContext_Make(reinterpret_cast(&src)); return handle(reinterpret_cast(tmp)); } }; @@ -164,8 +164,8 @@ template <> struct type_caster { PyObject *source = src.ptr(); if (PyObject_TypeCheck(source, &PySyclEventType)) { - DPCTLSyclEventRef ERef = - get_event_ref(reinterpret_cast(source)); + DPCTLSyclEventRef ERef = SyclEvent_GetEventRef( + reinterpret_cast(source)); sycl::event *ev = reinterpret_cast(ERef); value = *ev; return true; @@ -178,7 +178,7 @@ template <> struct type_caster static handle cast(sycl::event src, return_value_policy, handle) { - auto tmp = make_SyclEvent(reinterpret_cast(&src)); + auto tmp = SyclEvent_Make(reinterpret_cast(&src)); return handle(reinterpret_cast(tmp)); } }; diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 46be4c1c59..835fc3b2f4 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -751,24 +751,24 @@ def as_usm_memory(obj): ) -cdef api DPCTLSyclUSMRef Memory_get_usm_pointer(_Memory obj): +cdef api DPCTLSyclUSMRef Memory_GetUsmPointer(_Memory obj): "Pointer of USM allocation" return obj.memory_ptr -cdef api DPCTLSyclContextRef Memory_get_context_ref(_Memory obj): +cdef api DPCTLSyclContextRef Memory_GetContextRef(_Memory obj): "Context reference to which USM allocation is bound" return obj.queue._context.get_context_ref() -cdef api DPCTLSyclQueueRef Memory_get_queue_ref(_Memory obj): +cdef api DPCTLSyclQueueRef Memory_GetQueueRef(_Memory obj): """Queue associated with this allocation, used for copying, population, etc.""" return obj.queue.get_queue_ref() -cdef api size_t Memory_get_nbytes(_Memory obj): +cdef api size_t Memory_GetNumBytes(_Memory obj): "Size of the allocation in bytes." return obj.nbytes -cdef api object make_Memory( +cdef api object Memory_Make( DPCTLSyclUSMRef ptr, size_t nbytes, DPCTLSyclQueueRef QRef, diff --git a/dpctl/tests/test_sycl_context.py b/dpctl/tests/test_sycl_context.py index 1b67d1f4f3..7f0bc5d5ee 100644 --- a/dpctl/tests/test_sycl_context.py +++ b/dpctl/tests/test_sycl_context.py @@ -190,15 +190,15 @@ def test_context_repr(): assert type(ctx.__repr__()) is str -def test_cpython_api_get_context_ref(): +def test_cpython_api_SyclContext_GetContextRef(): import ctypes import sys ctx = dpctl.SyclContext() mod = sys.modules[ctx.__class__.__module__] - # get capsule storign get_context_ref function ptr - ctx_ref_fn_cap = mod.__pyx_capi__["get_context_ref"] - # construct Python callable to invoke "get_context_ref" + # get capsule storign SyclContext_GetContextRef function ptr + ctx_ref_fn_cap = mod.__pyx_capi__["SyclContext_GetContextRef"] + # construct Python callable to invoke "SyclContext_GetContextRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -213,15 +213,15 @@ def test_cpython_api_get_context_ref(): assert r1 == r2 -def test_cpython_api_make_SyclContext(): +def test_cpython_api_SyclContext_Make(): import ctypes import sys ctx = dpctl.SyclContext() mod = sys.modules[ctx.__class__.__module__] - # get capsule storign make_SyclContext function ptr - make_ctx_fn_cap = mod.__pyx_capi__["make_SyclContext"] - # construct Python callable to invoke "make_SyclContext" + # get capsule storign SyclContext_Make function ptr + make_ctx_fn_cap = mod.__pyx_capi__["SyclContext_Make"] + # construct Python callable to invoke "SyclContext_Make" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 329444ac90..409c5c5c13 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -731,15 +731,15 @@ def test_handle_no_device(): dpctl.select_device_with_aspects("cpu", excluded_aspects="cpu") -def test_cpython_api_get_device_ref(): +def test_cpython_api_SyclDevice_GetDeviceRef(): import ctypes import sys d = dpctl.SyclDevice() mod = sys.modules[d.__class__.__module__] - # get capsule storign get_device_ref function ptr - d_ref_fn_cap = mod.__pyx_capi__["get_device_ref"] - # construct Python callable to invoke "get_device_ref" + # get capsule storing SyclDevice_GetDeviceRef function ptr + d_ref_fn_cap = mod.__pyx_capi__["SyclDevice_GetDeviceRef"] + # construct Python callable to invoke "SyclDevice_GetDeviceRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -754,15 +754,15 @@ def test_cpython_api_get_device_ref(): assert r1 == r2 -def test_cpython_api_make_SyclDevice(): +def test_cpython_api_SyclDevice_Make(): import ctypes import sys d = dpctl.SyclDevice() mod = sys.modules[d.__class__.__module__] - # get capsule storign make_SyclContext function ptr - make_d_fn_cap = mod.__pyx_capi__["make_SyclDevice"] - # construct Python callable to invoke "make_SyclDevice" + # get capsule storign SyclContext_Make function ptr + make_d_fn_cap = mod.__pyx_capi__["SyclDevice_Make"] + # construct Python callable to invoke "SyclDevice_Make" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] diff --git a/dpctl/tests/test_sycl_event.py b/dpctl/tests/test_sycl_event.py index 12c48e48c7..e56337b7ff 100644 --- a/dpctl/tests/test_sycl_event.py +++ b/dpctl/tests/test_sycl_event.py @@ -234,15 +234,15 @@ def test_addressof_ref(): assert type(ref) is int -def test_cpython_api_get_event_ref(): +def test_cpython_api_SyclEvent_GetEventRef(): import ctypes import sys ev = dpctl.SyclEvent() mod = sys.modules[ev.__class__.__module__] - # get capsule storign get_event_ref function ptr - ev_ref_fn_cap = mod.__pyx_capi__["get_event_ref"] - # construct Python callable to invoke "get_event_ref" + # get capsule storign SyclEvent_GetEventRef function ptr + ev_ref_fn_cap = mod.__pyx_capi__["SyclEvent_GetEventRef"] + # construct Python callable to invoke "SyclEvent_GetEventRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -257,15 +257,15 @@ def test_cpython_api_get_event_ref(): assert r1 == r2 -def test_cpython_api_make_SyclEvent(): +def test_cpython_api_SyclEvent_Make(): import ctypes import sys ev = dpctl.SyclEvent() mod = sys.modules[ev.__class__.__module__] - # get capsule storing make_SyclEvent function ptr - make_e_fn_cap = mod.__pyx_capi__["make_SyclEvent"] - # construct Python callable to invoke "make_SyclDevice" + # get capsule storing SyclEvent_Make function ptr + make_e_fn_cap = mod.__pyx_capi__["SyclEvent_Make"] + # construct Python callable to invoke "SyclDevice_Make" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] diff --git a/dpctl/tests/test_sycl_queue.py b/dpctl/tests/test_sycl_queue.py index 6ad2571968..3d3a165d4b 100644 --- a/dpctl/tests/test_sycl_queue.py +++ b/dpctl/tests/test_sycl_queue.py @@ -469,12 +469,12 @@ def test_queue_capsule(): assert q2 != [] # compare with other types -def test_cpython_api_get_queue_ref(): +def test_cpython_api_SyclQueue_GetQueueRef(): q = dpctl.SyclQueue() mod = sys.modules[q.__class__.__module__] - # get capsule storign get_queue_ref function ptr - q_ref_fn_cap = mod.__pyx_capi__["get_queue_ref"] - # construct Python callable to invoke "get_queue_ref" + # get capsule storign SyclQueue_GetQueueRef function ptr + q_ref_fn_cap = mod.__pyx_capi__["SyclQueue_GetQueueRef"] + # construct Python callable to invoke "SyclQueue_GetQueueRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -489,12 +489,12 @@ def test_cpython_api_get_queue_ref(): assert r1 == r2 -def test_cpython_api_make_SyclQueue(): +def test_cpython_api_SyclQueue_Make(): q = dpctl.SyclQueue() mod = sys.modules[q.__class__.__module__] - # get capsule storing make_SyclQueue function ptr - make_SyclQueue_fn_cap = mod.__pyx_capi__["make_SyclQueue"] - # construct Python callable to invoke "make_SyclQueue" + # get capsule storing SyclQueue_Make function ptr + make_SyclQueue_fn_cap = mod.__pyx_capi__["SyclQueue_Make"] + # construct Python callable to invoke "SyclQueue_Make" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index 8ed932fa9d..9e728fb68b 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -530,11 +530,11 @@ def test_cpython_api(memory_ctor): mobj = memory_ctor(1024) mod = sys.modules[mobj.__class__.__module__] # get capsules storing function pointers - mem_ptr_fn_cap = mod.__pyx_capi__["Memory_get_usm_pointer"] - mem_q_ref_fn_cap = mod.__pyx_capi__["Memory_get_queue_ref"] - mem_ctx_ref_fn_cap = mod.__pyx_capi__["Memory_get_context_ref"] - mem_nby_fn_cap = mod.__pyx_capi__["Memory_get_nbytes"] - # construct Python callable to invoke "get_usm_pointer" + mem_ptr_fn_cap = mod.__pyx_capi__["Memory_GetUsmPointer"] + mem_q_ref_fn_cap = mod.__pyx_capi__["Memory_GetQueueRef"] + mem_ctx_ref_fn_cap = mod.__pyx_capi__["Memory_GetContextRef"] + mem_nby_fn_cap = mod.__pyx_capi__["Memory_GetNumBytes"] + # construct Python callable to invoke functions cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] From dae3902edb96c399d83021cec94c7e2b0ebb0382 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 11 Jan 2022 14:36:53 -0600 Subject: [PATCH 145/229] Renamed usm_ndarray's C-API functions as UsmNDArray_* --- dpctl/tensor/_usmarray.pyx | 14 +++++++------- dpctl/tests/test_usm_ndarray_ctor.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 2062cf9d67..a9d924da0f 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -1136,37 +1136,37 @@ cdef usm_ndarray _zero_like(usm_ndarray ary): return r -cdef api char* usm_ndarray_get_data(usm_ndarray arr): +cdef api char* UsmNDArray_GetData(usm_ndarray arr): """ """ return arr.get_data() -cdef api int usm_ndarray_get_ndim(usm_ndarray arr): +cdef api int UsmNDArray_GetNDim(usm_ndarray arr): """""" return arr.get_ndim() -cdef api Py_ssize_t* usm_ndarray_get_shape(usm_ndarray arr): +cdef api Py_ssize_t* UsmNDArray_GetShape(usm_ndarray arr): """ """ return arr.get_shape() -cdef api Py_ssize_t* usm_ndarray_get_strides(usm_ndarray arr): +cdef api Py_ssize_t* UsmNDArray_GetStrides(usm_ndarray arr): """ """ return arr.get_strides() -cdef api int usm_ndarray_get_typenum(usm_ndarray arr): +cdef api int UsmNDArray_GetTypenum(usm_ndarray arr): """ """ return arr.get_typenum() -cdef api int usm_ndarray_get_flags(usm_ndarray arr): +cdef api int UsmNDArray_GetFlags(usm_ndarray arr): """ """ return arr.get_flags() -cdef api c_dpctl.DPCTLSyclQueueRef usm_ndarray_get_queue_ref(usm_ndarray arr): +cdef api c_dpctl.DPCTLSyclQueueRef UsmNDArray_GetQueueRef(usm_ndarray arr): """ """ return arr.get_queue_ref() diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index e6cee4a3a9..410c3128e8 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -353,7 +353,7 @@ def test_pyx_capi_get_data(): X = dpt.usm_ndarray(17)[1::2] get_data_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_data", + "UsmNDArray_GetData", b"char *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -366,7 +366,7 @@ def test_pyx_capi_get_shape(): X = dpt.usm_ndarray(17)[1::2] get_shape_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_shape", + "UsmNDArray_GetShape", b"Py_ssize_t *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -379,7 +379,7 @@ def test_pyx_capi_get_strides(): X = dpt.usm_ndarray(17)[1::2] get_strides_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_strides", + "UsmNDArray_GetStrides", b"Py_ssize_t *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -395,7 +395,7 @@ def test_pyx_capi_get_ndim(): X = dpt.usm_ndarray(17)[1::2] get_ndim_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_ndim", + "UsmNDArray_GetNDim", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -406,7 +406,7 @@ def test_pyx_capi_get_typenum(): X = dpt.usm_ndarray(17)[1::2] get_typenum_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_typenum", + "UsmNDArray_GetTypenum", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -419,7 +419,7 @@ def test_pyx_capi_get_flags(): X = dpt.usm_ndarray(17)[1::2] get_flags_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_flags", + "UsmNDArray_GetFlags", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -431,7 +431,7 @@ def test_pyx_capi_get_queue_ref(): X = dpt.usm_ndarray(17)[1::2] get_queue_ref_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_queue_ref", + "UsmNDArray_GetQueueRef", b"DPCTLSyclQueueRef (struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) From e432c392de7a90fe8eb70b1d09286f6547cde099 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 12 Jan 2022 10:29:17 -0600 Subject: [PATCH 146/229] dpctl_capi imports tensor submodule as well as part of importing dpctl --- dpctl/apis/include/dpctl_capi.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpctl/apis/include/dpctl_capi.h b/dpctl/apis/include/dpctl_capi.h index 2be253a618..18b01f9e19 100644 --- a/dpctl/apis/include/dpctl_capi.h +++ b/dpctl/apis/include/dpctl_capi.h @@ -38,6 +38,8 @@ #include "../_sycl_queue_api.h" #include "../memory/_memory.h" #include "../memory/_memory_api.h" +#include "../tensor/_usmarray.h" +#include "../tensor/_usmarray_api.h" // clang-format on /* @@ -53,5 +55,6 @@ void import_dpctl(void) import_dpctl___sycl_event(); import_dpctl___sycl_queue(); import_dpctl__memory___memory(); + import_dpctl__tensor___usmarray(); return; } From a280da300648aeae22f76daebe6aa53f5ca70b55 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 12 Jan 2022 18:17:32 -0600 Subject: [PATCH 147/229] libsyclinterface/CMakeLists.txt should use CMAKE_CURRENT_SOURCE_DIR (#744) It should use CMAKE_CURRENT_SOURCE_DIR rather than CMAKE_SOURCE_DIR to work correctly if included as subdirectory from upper level. --- libsyclinterface/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index bb921b8603..3362e2b2a3 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -6,7 +6,7 @@ project( ) # Load our CMake modules to search for DPCPP and Level Zero -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/") find_package(Git REQUIRED) option(DPCTL_DPCPP_HOME_DIR @@ -69,8 +69,8 @@ if(DPCTL_ENABLE_LO_PROGRAM_CREATION) endif() configure_file( - ${CMAKE_SOURCE_DIR}/include/Config/dpctl_config.h.in - ${CMAKE_SOURCE_DIR}/include/Config/dpctl_config.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config/dpctl_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config/dpctl_config.h ) # Set the C++ standard to C++17 @@ -158,8 +158,8 @@ add_library(DPCTLSyclInterface target_include_directories(DPCTLSyclInterface PRIVATE - ${CMAKE_SOURCE_DIR}/include/ - ${CMAKE_SOURCE_DIR}/helper/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/helper/include/ ${IntelSycl_SYCL_INCLUDE_DIR} ) @@ -193,19 +193,19 @@ install(TARGETS ) # Install all headers -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/*.h") +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") foreach(HEADER ${HEADERS}) install(FILES "${HEADER}" DESTINATION include) endforeach() # Install all headers in include/Support -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/Support/*.h") +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") foreach(HEADER ${HEADERS}) install(FILES "${HEADER}" DESTINATION include/Support) endforeach() # Install all headers in include/Config -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/Config/*.h") +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") foreach(HEADER ${HEADERS}) install(FILES "${HEADER}" DESTINATION include/Config) endforeach() From 2614bb960aba9d00b18eee86c1f13913b9a35ed0 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Thu, 13 Jan 2022 23:35:53 -0600 Subject: [PATCH 148/229] Add a badge indicating status of sphinx generation (#745) Adds a badge to the README that indicates if the generate-docs workflow succeeded for `master`. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6d06fdf92d..031267f7e9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpctl/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpctl?branch=master) +![Generate Documentation](https://github.com/IntelPython/dpctl/actions/workflows/generate-docs.yml/badge.svg?branch=master) About ===== From b96a882129f08716ab77124c6a45c79c3cdedb4e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 14 Jan 2022 19:33:13 -0600 Subject: [PATCH 149/229] Added docstring as per PR feedback --- dpctl/memory/_memory.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 835fc3b2f4..c40cd0fb89 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -774,6 +774,7 @@ cdef api object Memory_Make( DPCTLSyclQueueRef QRef, object owner ): + "Create _Memory Python object from preallocated memory." return _Memory.create_from_usm_pointer_size_qref( ptr, nbytes, QRef, memory_owner=owner ) From 50f16f4f72022eef51b277b996c582432a94b94b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 14 Jan 2022 19:37:38 -0600 Subject: [PATCH 150/229] populated docstrings of C-API functions of usmarray --- dpctl/tensor/_usmarray.pyx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index a9d924da0f..959164dd3c 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -1137,36 +1137,35 @@ cdef usm_ndarray _zero_like(usm_ndarray ary): cdef api char* UsmNDArray_GetData(usm_ndarray arr): - """ - """ + """Get allocation pointer of zero index element of array """ return arr.get_data() cdef api int UsmNDArray_GetNDim(usm_ndarray arr): - """""" + """Get array rank: length of its shape""" return arr.get_ndim() cdef api Py_ssize_t* UsmNDArray_GetShape(usm_ndarray arr): - """ """ + """Get host pointer to shape vector""" return arr.get_shape() cdef api Py_ssize_t* UsmNDArray_GetStrides(usm_ndarray arr): - """ """ + """Get host pointer to strides vector""" return arr.get_strides() cdef api int UsmNDArray_GetTypenum(usm_ndarray arr): - """ """ + """Get type number for data type of array elements""" return arr.get_typenum() cdef api int UsmNDArray_GetFlags(usm_ndarray arr): - """ """ + """Get flags of array""" return arr.get_flags() cdef api c_dpctl.DPCTLSyclQueueRef UsmNDArray_GetQueueRef(usm_ndarray arr): - """ """ + """Get DPCTLSyclQueueRef for queue associated with the array""" return arr.get_queue_ref() From 5e56e8e6d66066e6869b074b0f31969afca47186 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 7 Jan 2022 12:18:56 -0600 Subject: [PATCH 151/229] Changed building to use scikit-build and cmake to build dpctl The change to _dlpack was to fix incorrect path to includes (relative to current dir). --- .gitignore | 1 + CMakeLists.txt | 25 ++ dpctl/CMakeLists.txt | 56 +++ dpctl/cmake/copy_generated_headers.cmake | 3 + dpctl/memory/CMakeLists.txt | 6 + dpctl/program/CMakeLists.txt | 6 + dpctl/tensor/CMakeLists.txt | 6 + dpctl/tensor/_dlpack.pxd | 2 +- dpctl/tensor/_dlpack.pyx | 2 +- dpctl/utils/CMakeLists.txt | 6 + libsyclinterface/CMakeLists.txt | 71 ++-- setup.py | 448 +---------------------- 12 files changed, 164 insertions(+), 468 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 dpctl/CMakeLists.txt create mode 100644 dpctl/cmake/copy_generated_headers.cmake create mode 100644 dpctl/memory/CMakeLists.txt create mode 100644 dpctl/program/CMakeLists.txt create mode 100644 dpctl/tensor/CMakeLists.txt create mode 100644 dpctl/utils/CMakeLists.txt diff --git a/.gitignore b/.gitignore index 97cccfe031..ae22979595 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__/ # CMake build and local install directory build +_skbuild build_cmake install diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..a8c03c3383 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) + +project(dpctl + LANGUAGES CXX + DESCRIPTION "Python interface for XPU programming" +) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(IntelDPCPP REQUIRED) + +add_subdirectory(libsyclinterface) + +file(GLOB _dpctl_capi_headers dpctl/apis/include/*.h*) +install(FILES ${_dpctl_capi_headers} + DESTINATION dpctl/include + COMPONENT DpctlCAPIHeaders +) + +add_subdirectory(dpctl) + +if (DPCTL_GENERATE_DOCS) + add_subdirectory(docs) +endif() diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt new file mode 100644 index 0000000000..f0240c79ee --- /dev/null +++ b/dpctl/CMakeLists.txt @@ -0,0 +1,56 @@ + +find_package(PythonExtensions REQUIRED) +find_package(NumPy REQUIRED) +find_package(Cython REQUIRED) + +# at build time create include/ directory and copy header files over +set(DPCTL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) +add_custom_target(_build_time_create_dpctl_include ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/libsyclinterface/include/* ${DPCTL_INCLUDE_DIR}/syclinterface + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/apis/include/* ${DPCTL_INCLUDE_DIR} + DEPENDS DPCTLSyclInterface +) + +set(CMAKE_INSTALL_RPATH "$ORIGIN") + +function(build_dpctl_ext _trgt _src _dest) + add_cython_target(${_trgt} ${_src} CXX OUTPUT_VAR _generated_src) + add_library(${_trgt} MODULE ${_generated_src}) + target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) + add_dependencies(${_trgt} _build_time_create_dpctl_include) + target_link_libraries(${_trgt} DPCTLSyclInterface) + python_extension_module(${_trgt}) + get_filename_component(_name_wle ${_generated_src} NAME_WLE) + get_filename_component(_generated_src_dir ${_generated_src} DIRECTORY) + set(_generated_public_h "${_generated_src_dir}/${_name_wle}.h") + set(_generated_api_h "${_generated_src_dir}/${_name_wle}_api.h") + set(_copy_trgt "${_trgt}_copy_capi_include") + add_custom_target( + ${_copy_trgt} ALL + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_public_h} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_generated_headers.cmake + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_api_h} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_generated_headers.cmake + DEPENDS ${_trgt} + VERBATIM + COMMENT "Copying Cython-generated headers to destination" + ) + install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest}) +endfunction() + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl") +endforeach() + +add_subdirectory(program) +add_subdirectory(memory) +add_subdirectory(tensor) +add_subdirectory(utils) diff --git a/dpctl/cmake/copy_generated_headers.cmake b/dpctl/cmake/copy_generated_headers.cmake new file mode 100644 index 0000000000..242dc29257 --- /dev/null +++ b/dpctl/cmake/copy_generated_headers.cmake @@ -0,0 +1,3 @@ +if (EXISTS ${SOURCE_FILE}) + configure_file(${SOURCE_FILE} ${DEST} COPYONLY) +endif() diff --git a/dpctl/memory/CMakeLists.txt b/dpctl/memory/CMakeLists.txt new file mode 100644 index 0000000000..c4f6d8469f --- /dev/null +++ b/dpctl/memory/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/memory") +endforeach() diff --git a/dpctl/program/CMakeLists.txt b/dpctl/program/CMakeLists.txt new file mode 100644 index 0000000000..f10706ed9d --- /dev/null +++ b/dpctl/program/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/program") +endforeach() diff --git a/dpctl/tensor/CMakeLists.txt b/dpctl/tensor/CMakeLists.txt new file mode 100644 index 0000000000..855bba7e8a --- /dev/null +++ b/dpctl/tensor/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/tensor") + target_include_directories(${_trgt} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) +endforeach() diff --git a/dpctl/tensor/_dlpack.pxd b/dpctl/tensor/_dlpack.pxd index 439880f7d2..58176ceb84 100644 --- a/dpctl/tensor/_dlpack.pxd +++ b/dpctl/tensor/_dlpack.pxd @@ -21,7 +21,7 @@ from ._usmarray cimport usm_ndarray -cdef extern from './include/dlpack/dlpack.h' nogil: +cdef extern from 'dlpack/dlpack.h' nogil: int device_CPU 'kDLCPU' int device_oneAPI 'kDLOneAPI' int device_OpenCL 'kDLOpenCL' diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 7d4ebfa0c3..fe762a016a 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -39,7 +39,7 @@ import dpctl import dpctl.memory as dpmem -cdef extern from './include/dlpack/dlpack.h' nogil: +cdef extern from 'dlpack/dlpack.h' nogil: cdef int DLPACK_VERSION cdef enum DLDeviceType: diff --git a/dpctl/utils/CMakeLists.txt b/dpctl/utils/CMakeLists.txt new file mode 100644 index 0000000000..11b0930052 --- /dev/null +++ b/dpctl/utils/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/utils") +endforeach() diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index 3362e2b2a3..6ec3314c56 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -125,7 +125,7 @@ elseif(UNIX) "-fsycl " ) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAGS}") - set(CMAKE_CXX_FLAGS "${CXXFLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXFLAGS}") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${CFLAGS} -ggdb3 -DDEBUG" ) @@ -157,8 +157,11 @@ add_library(DPCTLSyclInterface ) target_include_directories(DPCTLSyclInterface - PRIVATE + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/include/Support + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/helper/include/ ${IntelSycl_SYCL_INCLUDE_DIR} ) @@ -186,29 +189,55 @@ if(DPCTL_ENABLE_LO_PROGRAM_CREATION) ) endif() -install(TARGETS - DPCTLSyclInterface - LIBRARY - DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ -) - # Install all headers -file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include) -endforeach() -# Install all headers in include/Support -file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include/Support) -endforeach() +file(GLOB MAIN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") +file(GLOB SUPPORT_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") +file(GLOB CONFIG_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") -# Install all headers in include/Config -file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include/Config) +if (0) +set(ALL_HEADERS) +foreach(HEADER ${MAIN_HEADERS}) + list(APPEND ALL_HEADERS ${HEADER}) +endforeach() +foreach(HEADER ${SUPPORT_HEADERS}) + list(APPEND ALL_HEADERS ${HEADER}) endforeach() +foreach(HEADER ${CONFIG_HEADERS}) + list(APPEND ALL_HEADERS ${HEADER}) +endforeach() +endif() + +set_target_properties(DPCTLSyclInterface + PROPERTIES PUBLIC_HEADER + "${MAIN_HEADERS}" +) + +if (SKBUILD) + set(_lib_destination dpctl) + set(_include_destination dpctl/include/syclinterface) +else() + set(_lib_destination ${CMAKE_INSTALL_PREFIX}/lib) + set(_include_destination ${CMAKE_INSTALL_PREFIX}/include) +endif() + +install(TARGETS + DPCTLSyclInterface + LIBRARY + DESTINATION ${_lib_destination} + PUBLIC_HEADER + DESTINATION ${_include_destination} +) +install( + FILES ${SUPPORT_HEADERS} + DESTINATION ${_include_destination}/Support + COMPONENT SupportHeaders +) +install( + FILES ${CONFIG_HEADERS} + DESTINATION ${_include_destination}/Config + COMPONENT ConfigHeaders +) # Enable code coverage related settings if(DPCTL_GENERATE_COVERAGE) diff --git a/setup.py b/setup.py index d88580e242..8a2ba5726e 100644 --- a/setup.py +++ b/setup.py @@ -14,459 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob -import os -import os.path -import shutil -import sys - -import numpy as np -import setuptools.command.build_ext as orig_build_ext -import setuptools.command.develop as orig_develop -import setuptools.command.install as orig_install -from Cython.Build import cythonize -from setuptools import Extension, find_packages, setup +from setuptools import find_packages +from skbuild import setup import versioneer -IS_WIN = False -IS_LIN = False - -if "linux" in sys.platform: - IS_LIN = True -elif sys.platform in ["win32", "cygwin"]: - IS_WIN = True -else: - assert False, "We currently do not build for " + sys.platform - -# global variable used to pass value of --coverage option of develop command -# to build_ext command -_coverage = False -dpctl_sycl_interface_lib = "dpctl" -dpctl_sycl_interface_include = os.path.join("dpctl", "include") - # Get long description with open("README.md", "r", encoding="utf-8") as file: long_description = file.read() -def remove_empty(li): - return [el for el in li if el] - - -def get_sdl_cflags(): - cflags = [] - if IS_LIN: - cflags = [ - "-fstack-protector", - "-fPIC", - "-D_FORTIFY_SOURCE=2", - "-Wformat", - "-Wformat-security", - ] - # Add cflags from environment - cflags += remove_empty(os.getenv("CFLAGS", "").split(" ")) - - return cflags - - -def get_sdl_ldflags(): - ldflags = [] - if IS_LIN: - ldflags = ["-Wl,-z,noexecstack,-z,relro,-z,now"] - elif IS_WIN: - ldflags = [r"/NXCompat", r"/DynamicBase"] - # Add ldflags from environment - ldflags += remove_empty(os.getenv("LDFLAGS", "").split(" ")) - - return ldflags - - -def get_other_cxxflags(): - if IS_LIN: - return ["-O3", "-std=c++17"] - elif IS_WIN: - # FIXME: These are specific to MSVC and we should first make sure - # what compiler we are using. - return [r"/Ox", r"/std:c++17"] - - -def get_suppressed_warning_flags(): - if IS_LIN: - # PEP 590 renamed "tp_print" to "tp_vectorcall" and this causes a flood - # of deprecation warnings in the Cython generated module. This flag - # temporarily suppresses the warnings. The flag should not be needed - # once we move to Python 3.9 and/or Cython 0.30. - return ["-Wno-deprecated-declarations"] - elif IS_WIN: - return [] - - -def build_backend(l0_support, coverage, sycl_compiler_prefix): - import os.path - from importlib.util import module_from_spec, spec_from_file_location - - spec = spec_from_file_location( - "build_backend", os.path.join("scripts", "build_backend.py") - ) - builder_module = module_from_spec(spec) - spec.loader.exec_module(builder_module) - builder_module.build_backend( - l0_support=l0_support, - code_coverage=coverage, - sycl_compiler_prefix=sycl_compiler_prefix, - ) - - -def extensions(): - # Security flags - eca = get_sdl_cflags() - ela = get_sdl_ldflags() - libs = [] - libraries = [] - - if IS_LIN: - libs += ["rt", "DPCTLSyclInterface"] - libraries = [dpctl_sycl_interface_lib] - runtime_library_dirs = ["$ORIGIN"] - elif IS_WIN: - libs += ["DPCTLSyclInterface"] - libraries = [dpctl_sycl_interface_lib] - runtime_library_dirs = [] - - extension_args = { - "depends": [ - dpctl_sycl_interface_include, - ], - "include_dirs": [np.get_include(), dpctl_sycl_interface_include], - "extra_compile_args": ( - eca + get_other_cxxflags() + get_suppressed_warning_flags() - ), - "extra_link_args": ela, - "libraries": libs, - "library_dirs": libraries, - "runtime_library_dirs": runtime_library_dirs, - "language": "c++", - "define_macros": [], - } - - extensions = [ - Extension( - "dpctl._sycl_context", - [ - os.path.join("dpctl", "_sycl_context.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_device", - [ - os.path.join("dpctl", "_sycl_device.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_device_factory", - [ - os.path.join("dpctl", "_sycl_device_factory.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_event", - [ - os.path.join("dpctl", "_sycl_event.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_platform", - [ - os.path.join("dpctl", "_sycl_platform.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_queue", - [ - os.path.join("dpctl", "_sycl_queue.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_queue_manager", - [ - os.path.join("dpctl", "_sycl_queue_manager.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.memory._memory", - [ - os.path.join("dpctl", "memory", "_memory.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.program._program", - [ - os.path.join("dpctl", "program", "_program.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.utils._compute_follows_data", - [ - os.path.join("dpctl", "utils", "_compute_follows_data.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.tensor._usmarray", - [ - os.path.join("dpctl", "tensor", "_usmarray.pyx"), - ], - depends=extension_args["depends"] - + [os.path.join("libtensor", "include", "usm_array.hpp")], - language="c++", - include_dirs=( - extension_args["include_dirs"] - + [os.path.join("libtensor", "include")] - ), - extra_compile_args=extension_args["extra_compile_args"], - extra_link_args=extension_args["extra_link_args"], - libraries=extension_args["libraries"], - library_dirs=extension_args["library_dirs"], - runtime_library_dirs=extension_args["runtime_library_dirs"], - define_macros=extension_args["define_macros"], - ), - Extension( - "dpctl.tensor._dlpack", - [ - os.path.join("dpctl", "tensor", "_dlpack.pyx"), - ], - depends=extension_args["depends"], - language="c++", - include_dirs=extension_args["include_dirs"] - + [ - os.path.join("dpctl", "tensor"), - ], - extra_compile_args=extension_args["extra_compile_args"], - extra_link_args=extension_args["extra_link_args"], - libraries=extension_args["libraries"], - library_dirs=extension_args["library_dirs"], - runtime_library_dirs=extension_args["runtime_library_dirs"], - define_macros=extension_args["define_macros"], - ), - ] - return extensions - - -class build_ext(orig_build_ext.build_ext): - description = "Build dpctl native extensions" - - def finalize_options(self): - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - super().finalize_options() - - def run(self): - return super().run() - - -def get_build_py(orig_build_py): - class build_py(orig_build_py): - def run(self): - dpctl_src_dir = self.get_package_dir("dpctl") - dpctl_build_dir = os.path.join(self.build_lib, "dpctl") - os.makedirs(dpctl_build_dir, exist_ok=True) - if IS_LIN: - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.so*")): - # Check if the file already exists before copying. - # The check is needed when dealing with symlinks. - if not os.path.exists( - os.path.join(dpctl_build_dir, os.path.basename(fn)) - ): - shutil.copy( - src=fn, - dst=dpctl_build_dir, - follow_symlinks=False, - ) - elif IS_WIN: - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.lib")): - shutil.copy(src=fn, dst=dpctl_build_dir) - - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.dll")): - shutil.copy(src=fn, dst=dpctl_build_dir) - else: - raise NotImplementedError("Unsupported platform") - return super().run() - - return build_py - - -class install(orig_install.install): - description = "Installs dpctl into Python prefix" - user_options = orig_install.install.user_options + [ - ( - "level-zero-support=", - None, - "Whether to enable support for program creation " - "for Level-zero backend", - ), - ( - "sycl-compiler-prefix=", - None, - "Path to SYCL compiler installation. None means " - "read it off ONEAPI_ROOT environment variable or fail.", - ), - ] - - def initialize_options(self): - super().initialize_options() - self.level_zero_support = "True" - self.sycl_compiler_prefix = None - - def finalize_options(self): - if isinstance(self.level_zero_support, str): - self.level_zero_support = self.level_zero_support.capitalize() - if self.level_zero_support in ["True", "False", "0", "1"]: - self.level_zero_support = bool(eval(self.level_zero_support)) - else: - raise ValueError( - "--level-zero-support value is invalid, use True/False" - ) - if isinstance(self.sycl_compiler_prefix, str): - if not os.path.exists(os.path.join(self.sycl_compiler_prefix)): - raise ValueError( - "--sycl-compiler-prefix expects a path " - "to an existing directory" - ) - elif self.sycl_compiler_prefix is None: - pass - else: - raise ValueError( - "--sycl-compiler-prefix value is invalid, use a " - "path to compiler installation. To use oneAPI, use the " - "default value, but remember to activate the compiler " - "environment" - ) - super().finalize_options() - - def run(self): - build_backend(self.level_zero_support, False, self.sycl_compiler_prefix) - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - cythonize(self.distribution.ext_modules) - ret = super().run() - if IS_LIN: - dpctl_build_dir = os.path.join( - os.path.dirname(__file__), self.build_lib, "dpctl" - ) - dpctl_install_dir = os.path.join(self.install_libbase, "dpctl") - for fn in glob.glob( - os.path.join(dpctl_install_dir, "*DPCTLSyclInterface.so*") - ): - os.remove(fn) - shutil.copy( - src=os.path.join(dpctl_build_dir, os.path.basename(fn)), - dst=dpctl_install_dir, - follow_symlinks=False, - ) - return ret - - -class develop(orig_develop.develop): - description = "Installs dpctl in place" - user_options = orig_develop.develop.user_options + [ - ( - "level-zero-support=", - None, - "Whether to enable support for program creation " - "for Level-zero backend", - ), - ( - "sycl-compiler-prefix=", - None, - "Path to SYCL compiler installation. None means " - "read it off ONEAPI_ROOT environment variable or fail.", - ), - ( - "coverage=", - None, - "Whether to generate coverage report " - "when building the backend library", - ), - ] - - def initialize_options(self): - super().initialize_options() - self.level_zero_support = "True" - self.coverage = "False" - self.sycl_compiler_prefix = None - - def finalize_options(self): - if isinstance(self.level_zero_support, str): - self.level_zero_support = self.level_zero_support.capitalize() - if self.level_zero_support in ["True", "False", "0", "1"]: - self.level_zero_support = bool(eval(self.level_zero_support)) - else: - raise ValueError( - "--level-zero-support value is invalid, use True/False" - ) - if isinstance(self.coverage, str): - self.coverage = self.coverage.capitalize() - if self.coverage in ["True", "False", "0", "1"]: - self.coverage = bool(eval(self.coverage)) - global _coverage - _coverage = self.coverage - else: - raise ValueError("--coverage value is invalid, use True/False") - if isinstance(self.sycl_compiler_prefix, str): - if not os.path.exists(os.path.join(self.sycl_compiler_prefix)): - raise ValueError( - "--sycl-compiler-prefix expects a path " - "to an existing directory" - ) - elif self.sycl_compiler_prefix is None: - pass - else: - raise ValueError( - "--sycl-compiler-prefix value is invalid, use a " - "path to compiler installation. To use oneAPI, use the " - "default value, but remember to activate the compiler " - "environment" - ) - super().finalize_options() - - def run(self): - build_backend( - self.level_zero_support, self.coverage, self.sycl_compiler_prefix - ) - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - cythonize(self.distribution.ext_modules) - return super().run() - - def _get_cmdclass(): cmdclass = versioneer.get_cmdclass() - cmdclass["build_py"] = get_build_py(cmdclass["build_py"]) - cmdclass["install"] = install - cmdclass["develop"] = develop - cmdclass["build_ext"] = build_ext return cmdclass @@ -481,9 +40,8 @@ def _get_cmdclass(): author="Intel Corporation", url="https://github.com/IntelPython/dpctl", packages=find_packages(include=["*"]), - package_data={"dpctl": ["tests/*", "tests/helper/*.py"]}, + package_data={"dpctl": ["tests/*.*", "tests/helper/*.py"]}, include_package_data=True, - ext_modules=extensions(), zip_safe=False, setup_requires=["Cython"], install_requires=[ From 8e957b1cef2c2ff4e1c2a211a872f50f2837e2cc Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 14 Jan 2022 15:42:20 -0600 Subject: [PATCH 152/229] Use CMAKE_BINARY_DIR not CMAKE_CURRENT_BINARY_DIR to refer to location of checkout --- libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake b/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake index 83bb8e61a5..1b1cca582e 100644 --- a/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake +++ b/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake @@ -32,7 +32,7 @@ function(get_level_zero_headers) COMMAND ${GIT_EXECUTABLE} fetch RESULT_VARIABLE result ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -65,7 +65,7 @@ function(get_level_zero_headers) RESULT_VARIABLE result OUTPUT_VARIABLE latest_tag ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -81,7 +81,7 @@ function(get_level_zero_headers) COMMAND ${GIT_EXECUTABLE} checkout ${latest_tag} RESULT_VARIABLE result ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -95,7 +95,7 @@ function(get_level_zero_headers) # Populate the path to the headers find_path(LEVEL_ZERO_INCLUDE_DIR NAMES zet_api.h - PATHS ${CMAKE_CURRENT_BINARY_DIR}/level-zero/include + PATHS ${CMAKE_BINARY_DIR}/level-zero/include NO_DEFAULT_PATH NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_PATH From 93f4d02120e68e5de05dadfd5f1b91954ee1e37b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 23 Jan 2022 07:53:26 -0600 Subject: [PATCH 153/229] Activate internal API to make sycl::program available --- libsyclinterface/tests/dpcpp_kernels.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsyclinterface/tests/dpcpp_kernels.hpp b/libsyclinterface/tests/dpcpp_kernels.hpp index 099ea1a2da..f4c0934d78 100644 --- a/libsyclinterface/tests/dpcpp_kernels.hpp +++ b/libsyclinterface/tests/dpcpp_kernels.hpp @@ -1,4 +1,8 @@ #pragma once +#ifndef __SYCL_INTERNAL_API +// make sure that sycl::program is defined and implemented +#define __SYCL_INTERNAL_API +#endif #include namespace dpcpp_kernels From 771af53559b176c3424d985f86796900aaed0bd0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 23 Jan 2022 08:19:39 -0600 Subject: [PATCH 154/229] Replaced use of relative path in #include statements Modified CMakeLists.txt to add needed include directories for the dpctl_c_api_tests target --- libsyclinterface/helper/include/dpctl_utils_helper.h | 2 +- libsyclinterface/tests/CMakeLists.txt | 4 ++++ libsyclinterface/tests/test_helper.cpp | 2 +- libsyclinterface/tests/test_sycl_device_aspects.cpp | 2 +- libsyclinterface/tests/test_sycl_device_interface.cpp | 2 +- libsyclinterface/tests/test_sycl_device_manager.cpp | 2 +- libsyclinterface/tests/test_sycl_device_subdevices.cpp | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libsyclinterface/helper/include/dpctl_utils_helper.h b/libsyclinterface/helper/include/dpctl_utils_helper.h index e789c73517..57a308f5f0 100644 --- a/libsyclinterface/helper/include/dpctl_utils_helper.h +++ b/libsyclinterface/helper/include/dpctl_utils_helper.h @@ -24,8 +24,8 @@ #pragma once -#include "../include/dpctl_sycl_enum_types.h" #include "Support/DllExport.h" +#include "dpctl_sycl_enum_types.h" #include /*! diff --git a/libsyclinterface/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt index c4926ffa2d..0755fb9217 100644 --- a/libsyclinterface/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -51,6 +51,10 @@ if(DPCTL_GENERATE_COVERAGE) ${dpctl_sources} ${helper_sources} ) + target_include_directories(dpctl_c_api_tests + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../helper/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + ) target_link_libraries(dpctl_c_api_tests ${CMAKE_THREAD_LIBS_INIT} GTest::GTest diff --git a/libsyclinterface/tests/test_helper.cpp b/libsyclinterface/tests/test_helper.cpp index 76c8d50932..ae4f60b75c 100644 --- a/libsyclinterface/tests/test_helper.cpp +++ b/libsyclinterface/tests/test_helper.cpp @@ -24,8 +24,8 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "Config/dpctl_config.h" +#include "dpctl_utils_helper.h" #include #include #include diff --git a/libsyclinterface/tests/test_sycl_device_aspects.cpp b/libsyclinterface/tests/test_sycl_device_aspects.cpp index 8fdc1b34f6..90b29a8bac 100644 --- a/libsyclinterface/tests/test_sycl_device_aspects.cpp +++ b/libsyclinterface/tests/test_sycl_device_aspects.cpp @@ -1,8 +1,8 @@ -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_enum_types.h" +#include "dpctl_utils_helper.h" #include #include #include diff --git a/libsyclinterface/tests/test_sycl_device_interface.cpp b/libsyclinterface/tests/test_sycl_device_interface.cpp index b33e2c566f..a55bacecd7 100644 --- a/libsyclinterface/tests/test_sycl_device_interface.cpp +++ b/libsyclinterface/tests/test_sycl_device_interface.cpp @@ -24,11 +24,11 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_platform_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include diff --git a/libsyclinterface/tests/test_sycl_device_manager.cpp b/libsyclinterface/tests/test_sycl_device_manager.cpp index a7ca5ef6e1..87a5b1e6d9 100644 --- a/libsyclinterface/tests/test_sycl_device_manager.cpp +++ b/libsyclinterface/tests/test_sycl_device_manager.cpp @@ -24,11 +24,11 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_manager.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include diff --git a/libsyclinterface/tests/test_sycl_device_subdevices.cpp b/libsyclinterface/tests/test_sycl_device_subdevices.cpp index 14252bdc88..41cef23784 100644 --- a/libsyclinterface/tests/test_sycl_device_subdevices.cpp +++ b/libsyclinterface/tests/test_sycl_device_subdevices.cpp @@ -25,13 +25,13 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_enum_types.h" #include "dpctl_sycl_platform_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include From 18727b32b01d7e9d895771ffbee1248a71a79c89 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 24 Jan 2022 06:11:13 -0600 Subject: [PATCH 155/229] Ensure Cython files are built with security flags like before --- dpctl/CMakeLists.txt | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index f0240c79ee..c858afc7ef 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -3,6 +3,68 @@ find_package(PythonExtensions REQUIRED) find_package(NumPy REQUIRED) find_package(Cython REQUIRED) +if(WIN32) + string(CONCAT WARNING_FLAGS + "-Wall " + "-Wextra " + "-Winit-self " + "-Wunused-function " + "-Wuninitialized " + "-Wmissing-declarations " + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Ox ${WARNING_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Ox ${WARNING_FLAGS}") + set(CMAKE_C_FLAGS_DEBUG + "${CMAKE_C_FLAGS_DEBUG} ${WARNING_FLAGS} -ggdb3 -DDEBUG" + ) + set(CMAKE_CXX_FLAGS_DEBUG + "${CMAKE_CXX_FLAGS_DEBUG} ${WARNING_FLAGS} -ggdb3 -DDEBUG" + ) + set(DPCTL_LDFLAGS "/NXCompat /DynamicBase") +elseif(UNIX) + string(CONCAT WARNING_FLAGS + "-Wall " + "-Wextra " + "-Winit-self " + "-Wunused-function " + "-Wuninitialized " + "-Wmissing-declarations " + "-fdiagnostics-color=auto " + "-Wno-deprecated-declarations " + ) + string(CONCAT SDL_FLAGS + "-fstack-protector " + "-fstack-protector-all " + "-fpic " + "-fPIC " + "-D_FORTIFY_SOURCE=2 " + "-Wformat " + "-Wformat-security " + "-fno-strict-overflow " + "-fno-delete-null-pointer-checks " + ) + string(CONCAT CFLAGS + "${WARNING_FLAGS}" + "${SDL_FLAGS}" + ) + string(CONCAT CXXFLAGS + "${WARNING_FLAGS}" + "${SDL_FLAGS}" + "-fsycl " + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 ${CFLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 ${CXXFLAGS}") + set(CMAKE_C_FLAGS_DEBUG + "${CMAKE_C_FLAGS_DEBUG} ${CFLAGS} -ggdb3 -DDEBUG" + ) + set(CMAKE_CXX_FLAGS_DEBUG + "${CMAKE_CXX_FLAGS_DEBUG} ${CXXFLAGS} -ggdb3 -DDEBUG" + ) + set(DPCTL_LDFLAGS "-z,noexecstack,-z,relro,-z,now") +else() + message(FATAL_ERROR "Unsupported system.") +endif() + # at build time create include/ directory and copy header files over set(DPCTL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) add_custom_target(_build_time_create_dpctl_include ALL @@ -21,6 +83,7 @@ function(build_dpctl_ext _trgt _src _dest) target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) add_dependencies(${_trgt} _build_time_create_dpctl_include) target_link_libraries(${_trgt} DPCTLSyclInterface) + target_link_options(${_trgt} PRIVATE "LINKER:${DPCTL_LDFLAGS}") python_extension_module(${_trgt}) get_filename_component(_name_wle ${_generated_src} NAME_WLE) get_filename_component(_generated_src_dir ${_generated_src} DIRECTORY) From 1f6b40f68005566fb3d175c2c3f3e168671d0055 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 24 Jan 2022 07:21:56 -0600 Subject: [PATCH 156/229] Target check launches ctest with DPCTL_VERBOSITY=warning This is needed so that the test checking for captured cerr stream does not fail. Removed dpcpp_kernels.cpp file (dpcpp linker fails on it) Used -fno-sycl-use-footer to work-around a bug getting in a way of collecting profiling data --- libsyclinterface/tests/CMakeLists.txt | 10 +-- libsyclinterface/tests/dpcpp_kernels.cpp | 89 ------------------------ 2 files changed, 2 insertions(+), 97 deletions(-) delete mode 100644 libsyclinterface/tests/dpcpp_kernels.cpp diff --git a/libsyclinterface/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt index 0755fb9217..dc180ec70d 100644 --- a/libsyclinterface/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -4,7 +4,7 @@ find_package(Threads REQUIRED) # Emulate autotools like make check target to build tests set(CMAKE_CTEST_COMMAND ctest --progress --output-on-failure -j 4) -add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND}) +add_custom_target(check COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=warning ${CMAKE_CTEST_COMMAND}) enable_testing() include_directories( @@ -35,12 +35,11 @@ if(DPCTL_GENERATE_COVERAGE) list(REMOVE_ITEM dpctl_sources "${CMAKE_CURRENT_SOURCE_DIR}/../source/dpctl_vector_templ.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../source/dpcpp_kernels.cpp" ) # Add profiling flags set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping -DDPCTL_COVERAGE" + "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping -fno-sycl-use-footer -DDPCTL_COVERAGE" ) # Add all dpctl sources into a single executable so that we can run coverage @@ -94,10 +93,6 @@ if(DPCTL_GENERATE_COVERAGE) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) else() - add_library(dpcpp_kernels - STATIC - ${CMAKE_CURRENT_SOURCE_DIR}/dpcpp_kernels.cpp - ) file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) add_executable(dpctl_c_api_tests EXCLUDE_FROM_ALL ${sources}) target_link_libraries(dpctl_c_api_tests @@ -105,7 +100,6 @@ else() GTest::GTest DPCTLSyclInterface ${LEVEL_ZERO_LIBRARY} - dpcpp_kernels ) endif() diff --git a/libsyclinterface/tests/dpcpp_kernels.cpp b/libsyclinterface/tests/dpcpp_kernels.cpp deleted file mode 100644 index 897aa86de4..0000000000 --- a/libsyclinterface/tests/dpcpp_kernels.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "dpcpp_kernels.hpp" -#include -#include - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, int *, int); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, - size_t, - unsigned int *, - unsigned int); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, double *, double); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, float *, float); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, int *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, - size_t, - unsigned int *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, float *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, double *); - -template sycl::kernel dpcpp_kernels::get_mad_kernel(sycl::queue &, - size_t, - int *, - int *, - int *, - int); - -template sycl::kernel -dpcpp_kernels::get_mad_kernel(sycl::queue &, - size_t, - unsigned int *, - unsigned int *, - unsigned int *, - unsigned int); - -template sycl::kernel dpcpp_kernels::get_local_sort_kernel(sycl::queue &, - size_t, - size_t, - int *, - size_t); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - int *, - size_t, - int, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - unsigned int *, - size_t, - unsigned int, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - float *, - size_t, - float, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - double *, - size_t, - double, - int *); From 49f111c485552e0aae785dee378f764608e6f882 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 24 Jan 2022 12:11:02 -0600 Subject: [PATCH 157/229] Use cmake_process_manifest_hook setup argument to clean destination folder Delete DPCTLSyclInterface library shared objects from dpctl/ folder from previous runs. Monkey-patch skbuild.setuptools._copy_file to not follows symbolic links, to ensure that versioned DPCTLSyclInterface ends up as the proper set of symbolic links pointing to a single physical shared object file. If symbolic links are followed, 3 identical copies of the library end up in the installation folder. Downstream dpctl users may end up with several instances of the library in their process space which may end up with errors is statuful functinality (queue manager) is used. --- setup.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8a2ba5726e..a63fdba8ed 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path +import pathlib +import shutil + +import skbuild +import skbuild.setuptools_wrap +import skbuild.utils from setuptools import find_packages -from skbuild import setup import versioneer @@ -29,7 +35,51 @@ def _get_cmdclass(): return cmdclass -setup( +def cleanup_destination(cmake_manifest): + """Delete library files from dpctl/ folder before + letting skbuild copy them over to avoid errors. + """ + _to_unlink = [] + for fn in cmake_manifest: + bn = os.path.basename(fn) + # delete + if "DPCTLSyclInterface" in bn: + lib_fn = os.path.join("dpctl", bn) + if os.path.exists(lib_fn): + _to_unlink.append(lib_fn) + for fn in _to_unlink: + pathlib.Path(fn).unlink() + return cmake_manifest + + +def _patched_copy_file(src_file, dest_file, hide_listing=True): + """Copy ``src_file`` to ``dest_file`` ensuring parent directory exists. + + By default, message like `creating directory /path/to/package` and + `copying directory /src/path/to/package -> path/to/package` are displayed + on standard output. Setting ``hide_listing`` to False avoids message from + being displayed. + + NB: Patched here to not follows symbolic links + """ + # Create directory if needed + dest_dir = os.path.dirname(dest_file) + if dest_dir != "" and not os.path.exists(dest_dir): + if not hide_listing: + print("creating directory {}".format(dest_dir)) + skbuild.utils.mkdir_p(dest_dir) + + # Copy file + if not hide_listing: + print("copying {} -> {}".format(src_file, dest_file)) + shutil.copyfile(src_file, dest_file, follow_symlinks=False) + shutil.copymode(src_file, dest_file, follow_symlinks=False) + + +skbuild.setuptools_wrap._copy_file = _patched_copy_file + + +skbuild.setup( name="dpctl", version=versioneer.get_version(), cmdclass=_get_cmdclass(), @@ -54,4 +104,5 @@ def _get_cmdclass(): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", ], + cmake_process_manifest_hook=cleanup_destination, ) From 4a51351919e5cb03d5b6a72aacbda4a25e6d0cb0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 24 Jan 2022 15:17:40 -0600 Subject: [PATCH 158/229] Add build-time dependency on scikit-build, change build scripts --- conda-recipe/bld.bat | 2 +- conda-recipe/build.sh | 4 +++- conda-recipe/meta.yaml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 1ece7b30da..a3cd4a4f70 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -4,7 +4,7 @@ set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" "%PYTHON%" setup.py clean --all -set "INSTALL_CMD=install --sycl-compiler-prefix=%BUILD_PREFIX%\Library" +set "INSTALL_CMD=install -- -G Ninja -DDPCTL_DPCPP_HOME_DIR=%BUILD_PREFIX%\Library -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON" if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( rem Install and assemble wheel package from the build bits diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 777bc6f513..41f2546eb1 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -6,7 +6,9 @@ export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib" ${PYTHON} setup.py clean --all -INSTALL_CMD="install --sycl-compiler-prefix=$BUILD_PREFIX" +export CMAKE_GENERATOR="Unix Makefiles" +INSTALL_CMD="install -- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX}" +echo "${PYTHON} setup.py ${INSTALL_CMD}" # Workaround for: # DPC++ launched by cmake does not see components of `dpcpp_cpp_rt`, diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 08170e4d3a..ab01e3c178 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -22,12 +22,13 @@ requirements: - python - make # [unix] - ninja # [win] + - scikit-build - numpy 1.19 - wheel run: - python - {{ pin_compatible('numpy') }} - - dpcpp-cpp-rt >=2021.2 + - dpcpp-cpp-rt >=2022.0 test: requires: From a83d25ad6cb97c580091012e8943c90217232c7c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 25 Jan 2022 11:04:47 -0600 Subject: [PATCH 159/229] Fixed GetNumDevices tests, added support for conversion of DPCTL_ALL_BACKENDS to sycl --- .../helper/source/dpctl_utils_helper.cpp | 2 + .../tests/test_sycl_device_manager.cpp | 59 +++++++++++++------ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp index 212a5634fa..bf3f590bc6 100644 --- a/libsyclinterface/helper/source/dpctl_utils_helper.cpp +++ b/libsyclinterface/helper/source/dpctl_utils_helper.cpp @@ -96,6 +96,8 @@ backend DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType BeTy) return backend::level_zero; case DPCTLSyclBackendType::DPCTL_OPENCL: return backend::opencl; + case DPCTLSyclBackendType::DPCTL_ALL_BACKENDS: + return backend::all; default: throw std::runtime_error("Unsupported backend type"); } diff --git a/libsyclinterface/tests/test_sycl_device_manager.cpp b/libsyclinterface/tests/test_sycl_device_manager.cpp index 87a5b1e6d9..77b7a21b07 100644 --- a/libsyclinterface/tests/test_sycl_device_manager.cpp +++ b/libsyclinterface/tests/test_sycl_device_manager.cpp @@ -139,18 +139,30 @@ INSTANTIATE_TEST_SUITE_P( struct TestGetNumDevicesForDTy : public ::testing::TestWithParam { - size_t nDevices = 0; + int param; + sycl::info::device_type sycl_dty; + TestGetNumDevicesForDTy() { - cl::sycl::info::device_type sycl_dty = - DPCTL_DPCTLDeviceTypeToSyclDeviceType( - DPCTLSyclDeviceType(GetParam())); - - auto devices = cl::sycl::device::get_devices(sycl_dty); - EXPECT_TRUE(devices.size() == DPCTLDeviceMgr_GetNumDevices(GetParam())); + param = GetParam(); + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType(param); + sycl_dty = DPCTL_DPCTLDeviceTypeToSyclDeviceType(DTy); } }; +TEST_P(TestGetNumDevicesForDTy, ChkGetNumDevices) +{ + auto devices = sycl::device::get_devices(sycl_dty); + size_t nDevices = 0; + sycl::default_selector mRanker; + for (const sycl::device &d : devices) { + if (mRanker(d) < 0) + continue; + ++nDevices; + } + EXPECT_TRUE(nDevices == DPCTLDeviceMgr_GetNumDevices(param)); +} + INSTANTIATE_TEST_SUITE_P( GetDevices, TestGetNumDevicesForDTy, @@ -162,22 +174,33 @@ INSTANTIATE_TEST_SUITE_P( struct TestGetNumDevicesForBTy : public ::testing::TestWithParam { - size_t nDevices = 0; + int param; + sycl::backend sycl_bty; TestGetNumDevicesForBTy() { - cl::sycl::backend sycl_bty = DPCTL_DPCTLBackendTypeToSyclBackend( - DPCTLSyclBackendType(GetParam())); - - auto platforms = cl::sycl::platform::get_platforms(); - for (const auto &P : platforms) { - if (P.get_backend() == sycl_bty) { - auto devices = P.get_devices(); - EXPECT_TRUE(devices.size() == - DPCTLDeviceMgr_GetNumDevices(GetParam())); + param = GetParam(); + sycl_bty = + DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType(param)); + } +}; + +TEST_P(TestGetNumDevicesForBTy, ChkGetNumDevices) +{ + auto platforms = cl::sycl::platform::get_platforms(); + size_t nDevices = 0; + sycl::default_selector mRanker; + for (const auto &P : platforms) { + if ((P.get_backend() == sycl_bty) || (sycl_bty == sycl::backend::all)) { + auto devices = P.get_devices(); + for (const sycl::device &d : devices) { + if (mRanker(d) < 0) + continue; + ++nDevices; } } } -}; + EXPECT_TRUE(nDevices == DPCTLDeviceMgr_GetNumDevices(param)); +} INSTANTIATE_TEST_SUITE_P( GetDevices, From 571fe43c9771c9aa6c0a3ae7feb1579686dccbf1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 24 Jan 2022 15:44:18 -0600 Subject: [PATCH 160/229] Generate coverage workflow uses scikit-build --- .github/workflows/generate-coverage.yaml | 34 ++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index 776c1316fb..d21da02645 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -11,7 +11,7 @@ jobs: env: ONEAPI_ROOT: /opt/intel/oneapi - GTEST_ROOT: /home/runner/work/googletest-release-1.10.0/install + GTEST_ROOT: /home/runner/work/googletest-release-1.11.0/install steps: - name: Cancel Previous Runs @@ -29,8 +29,8 @@ jobs: - name: Install Intel OneAPI run: | - sudo apt-get install intel-oneapi-compiler-dpcpp-cpp=2021.3.0-3350 - sudo apt-get install intel-oneapi-tbb=2021.3.0-511 + sudo apt-get install intel-oneapi-compiler-dpcpp-cpp + sudo apt-get install intel-oneapi-tbb - name: Install CMake run: | @@ -39,7 +39,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' architecture: x64 - name: Cache Gtest @@ -47,8 +47,8 @@ jobs: uses: actions/cache@v2 with: path: | - /home/runner/work/googletest-release-1.10.0/install - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-release-1.10.0/install/include/gtest/*') }} + /home/runner/work/googletest-release-1.11.0/install + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-release-1.11.0/install/include/gtest/*') }} restore-keys: | ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- @@ -59,12 +59,12 @@ jobs: shell: bash -l {0} run: | cd /home/runner/work - wget https://github.com/google/googletest/archive/refs/tags/release-1.10.0.tar.gz - tar xf release-1.10.0.tar.gz - cd googletest-release-1.10.0 + wget https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz + tar xf release-1.11.0.tar.gz + cd googletest-release-1.11.0 mkdir build cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-release-1.10.0/install + cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-release-1.11.0/install make && make install - name: Checkout repo @@ -79,14 +79,19 @@ jobs: - name: Install dpctl dependencies shell: bash -l {0} run: | - pip install numpy cython setuptools pytest pytest-cov coverage[toml] + pip install numpy cython setuptools pytest pytest-cov scikit-build coverage[toml] - name: Build dpctl with coverage shell: bash -l {0} run: | source /opt/intel/oneapi/setvars.sh - python setup.py develop --coverage=True - python -c "import dpctl; print(dpctl.__version__); dpctl.lsplatform()" + export _SAVED_PATH=${PATH} + export PATH=$(dirname $(dirname $(which icx)))/bin-llvm:${PATH} + python setup.py develop -- -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_GENERATE_COVERAGE=ON -DDPCTL_BUILD_CAPI_TESTS=ON -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) + make -C $(find _skbuild -name tests) lcov-genhtml + export PATH=${_SAVED_PATH} + unset _SAVED_PATH + python -c "import dpctl; print(dpctl.__version__); dpctl.lsplatform()" || exit 1 pytest -q -ra --disable-warnings --cov-config pyproject.toml --cov dpctl --cov-report term-missing --pyargs dpctl -vv - name: Install coverall dependencies @@ -96,8 +101,9 @@ jobs: pip install coveralls==3.2.0 - name: Upload coverage data to coveralls.io + shell: bash -l {0} run: | - coveralls-lcov -v -n build_cmake/tests/dpctl.lcov > dpctl-c-api-coverage.json + coveralls-lcov -v -n $(find _skbuild -name tests)/dpctl.lcov > dpctl-c-api-coverage.json coveralls --service=github --merge=dpctl-c-api-coverage.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 25332b30aa53ad5705b8526938ea1d3d3baab9c1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 25 Jan 2022 13:49:45 -0600 Subject: [PATCH 161/229] Vendor IntelDPCPPConfig.cmake Two reasons to do this: 1. to allow for providing hint env variables to help cmake find `SYCL_INCLUDE_DIR` and `SYCL_LIBRARY_DIR` for custom layouts of DPC++ installed into Python environment with conda 2. To allow use of nightly sycl bundles to build dpctl (bundle does not include IntelDPCPPConig.cmake) --- CMakeLists.txt | 2 +- cmake/IntelDPCPPConfig.cmake | 293 +++++++++++++++++++++++++++++++++++ 2 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 cmake/IntelDPCPPConfig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c03c3383..e215bbbcce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ project(dpctl set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) -find_package(IntelDPCPP REQUIRED) +find_package(IntelDPCPP REQUIRED PATHS ${CMAKE_SOURCE_DIR}/cmake NO_DEFAULT_PATH) add_subdirectory(libsyclinterface) diff --git a/cmake/IntelDPCPPConfig.cmake b/cmake/IntelDPCPPConfig.cmake new file mode 100644 index 0000000000..8f683006be --- /dev/null +++ b/cmake/IntelDPCPPConfig.cmake @@ -0,0 +1,293 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +IntelDPCPPConfig +------- + +DPCPP Library to verify DPCPP/SYCL compatability of CMAKE_CXX_COMPILER +and passes relevant compiler flags. + +Result Variables +^^^^^^^^^^^^^^^^ + +This will define the following variables: + +``IntelDPCPP_FOUND`` + True if the system has the DPCPP library. +``SYCL_LANGUAGE_VERSION`` + The SYCL language spec version by Compiler. +``SYCL_INCLUDE_DIR`` + Include directories needed to use SYCL. +``SYCL_IMPLEMENTATION_ID`` + The SYCL compiler variant. +``SYCL_FLAGS`` + SYCL specific flags for the compiler. + +Cache Variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``SYCL_INCLUDE_DIR`` + The directory containing ``sycl.hpp``. +``SYCL_LIBRARY_DIR`` + The path to the SYCL library. +``SYCL_FLAGS`` + SYCL specific flags for the compiler. +``SYCL_LANGUAGE_VERSION`` + The SYCL language spec version by Compiler. + + +.. note:: + + For now, user needs to set -DCMAKE_CXX_COMPILER or environment of + CXX pointing to SYCL compatible compiler ( eg: icx, clang++, icpx) + + Note: do not set to DPCPP compiler. If set to a Compiler family + that supports dpcpp ( eg: IntelLLVM) both DPCPP and SYCL + features are enabled. + + And add this package to user's Cmake config file. + + .. code-block:: cmake + + find_package(IntelDPCPP REQUIRED) + +#]=======================================================================] + +include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) + +find_package(PkgConfig QUIET) +if(PKG_CONFIG_FOUND) + # TODO add dependency package module checks, if any +endif() + + +# TODO: can't use find_program to override the CMAKE_CXX_COMPILER as +# Platform/ files are executed, potentially for a different compiler. +# Safer approach is to make user to define CMAKE_CXX_COMPILER. + +string(COMPARE EQUAL "${CMAKE_CXX_COMPILER}" "" nocmplr) +if(nocmplr) + set(IntelDPCPP_FOUND False) + set(SYCL_REASON_FAILURE "SYCL: CMAKE_CXX_COMPILER not set!!") + set(IntelDPCPP_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}") +endif() + +# Check for known compiler family that supports SYCL + +if( NOT "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xClang" AND + NOT "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xIntelLLVM") + set(IntelDPCPP_FOUND False) + set(SYCL_REASON_FAILURE "Unsupported compiler family ${CMAKE_CXX_COMPILER_ID} and compiler ${CMAKE_CXX_COMPILER}!!") + set(IntelDPCPP_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}") + return() +endif() + +# Assume that CXX Compiler supports SYCL and then test to verify. +set(SYCL_COMPILER ${CMAKE_CXX_COMPILER}) + + +# Function to write a test case to verify SYCL features. + +function(SYCL_FEATURE_TEST_WRITE src) + + set(pp_if "#if") + set(pp_endif "#endif") + + set(SYCL_TEST_CONTENT "") + string(APPEND SYCL_TEST_CONTENT "#include \nusing namespace std;\n") + string(APPEND SYCL_TEST_CONTENT "int main(){\n") + + # Feature tests goes here + + string(APPEND SYCL_TEST_CONTENT "${pp_if} defined(SYCL_LANGUAGE_VERSION)\n") + string(APPEND SYCL_TEST_CONTENT "cout << \"SYCL_LANGUAGE_VERSION=\"< Date: Tue, 25 Jan 2022 13:52:52 -0600 Subject: [PATCH 162/229] os-llvm-sycl-build workload uses scikit-build --- .github/workflows/os-llvm-sycl-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index 47b183cd7e..8268bd0683 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -79,7 +79,7 @@ jobs: - name: Install dpctl dependencies shell: bash -l {0} run: | - pip install numpy cython setuptools pytest + pip install numpy cython setuptools pytest scikit-build - name: Checkout repo uses: actions/checkout@v2 @@ -97,5 +97,6 @@ jobs: export OCL_ICD_FILENAMES=libintelocl.so:libintelocl_emu.so clang++ --version sycl-ls - python setup.py develop --sycl-compiler-prefix=$(dirname $(dirname `which clang++`)) - python -m pytest -v dpctl/tests + python setup.py develop -- -G "Unix Makefiles" -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=$(dirname $(dirname $(which clang))) -DDPCTL_DPCPP_FROM_ONEAPI=OFF + python -c "import dpctl; dpctl.lsplatform()" || exit 1 + SYCL_ENABLE_HOST_DEVICE=1 python -m pytest -v dpctl/tests From 2e3f5cd9d9be00f6584276d18dafcdaf0fe83ca9 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 25 Jan 2022 14:24:56 -0600 Subject: [PATCH 163/229] Generate docs uses scikit-built setup command --- .github/workflows/generate-docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 3f81a5eadc..052a82a304 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -49,7 +49,7 @@ jobs: if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | - pip install numpy cython setuptools sphinx sphinx_rtd_theme pydot graphviz sphinxcontrib-programoutput + pip install numpy cython setuptools scikit-build sphinx sphinx_rtd_theme pydot graphviz sphinxcontrib-programoutput - name: Checkout repo uses: actions/checkout@v2 with: @@ -60,8 +60,8 @@ jobs: shell: bash -l {0} run: | source /opt/intel/oneapi/setvars.sh - python setup.py develop - python -c "import dpctl; print(dpctl.__version__)" + python setup.py develop -- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX} + python -c "import dpctl; print(dpctl.__version__)" || exit 1 - name: Build docs if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} From d6618d6cd47d8550083bcb2871aadecba952937a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 Jan 2022 09:46:52 -0600 Subject: [PATCH 164/229] Make the build work on Windows --- CMakeLists.txt | 2 +- cmake/IntelDPCPPConfig.cmake | 4 ++-- conda-recipe/bld.bat | 7 ++++--- conda-recipe/build.sh | 8 ++++---- conda-recipe/meta.yaml | 2 +- dpctl/CMakeLists.txt | 33 +++++++++++++++++++++++++++++++-- libsyclinterface/CMakeLists.txt | 20 ++++++-------------- 7 files changed, 49 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e215bbbcce..8aff7d1d70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.21 FATAL_ERROR) +cmake_minimum_required(VERSION 3.21...3.22 FATAL_ERROR) project(dpctl LANGUAGES CXX diff --git a/cmake/IntelDPCPPConfig.cmake b/cmake/IntelDPCPPConfig.cmake index 8f683006be..2b73830f1e 100644 --- a/cmake/IntelDPCPPConfig.cmake +++ b/cmake/IntelDPCPPConfig.cmake @@ -201,7 +201,7 @@ if(SYCL_COMPILER) NAMES include HINTS - ${SYCL_PACKAGE_DIR} ENV SYCL_INCLUDE_DIR_HINT + ${SYCL_PACKAGE_DIR} $ENV{SYCL_INCLUDE_DIR_HINT} NO_DEFAULT_PATH ) @@ -210,7 +210,7 @@ if(SYCL_COMPILER) NAMES lib lib64 HINTS - ${SYCL_PACKAGE_DIR} ENV SYCL_LIBRARY_DIR_HINT + ${SYCL_PACKAGE_DIR} $ENV{SYCL_LIBRARY_DIR_HINT} NO_DEFAULT_PATH ) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index a3cd4a4f70..ee34945ea7 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -4,16 +4,17 @@ set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" "%PYTHON%" setup.py clean --all -set "INSTALL_CMD=install -- -G Ninja -DDPCTL_DPCPP_HOME_DIR=%BUILD_PREFIX%\Library -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON" +set "SKBUILD_ARGS=-- -G Ninja -DDPCTL_DPCPP_HOME_DIR=%BUILD_PREFIX%\Library -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON" +set "SYCL_INCLUDE_DIR_HINT=%BUILD_PREFIX%\Library\lib\clang\14.0.0" if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( rem Install and assemble wheel package from the build bits - "%PYTHON%" setup.py %INSTALL_CMD% bdist_wheel + "%PYTHON%" setup.py install bdist_wheel %SKBUILD_ARGS% if errorlevel 1 exit 1 copy dist\dpctl*.whl %WHEELS_OUTPUT_FOLDER% if errorlevel 1 exit 1 ) ELSE ( rem Only install - "%PYTHON%" setup.py %INSTALL_CMD% + "%PYTHON%" setup.py install %SKBUILD_ARGS% if errorlevel 1 exit 1 ) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 41f2546eb1..9d581001fe 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -7,8 +7,8 @@ export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib" ${PYTHON} setup.py clean --all export CMAKE_GENERATOR="Unix Makefiles" -INSTALL_CMD="install -- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX}" -echo "${PYTHON} setup.py ${INSTALL_CMD}" +SKBUILD_ARGS="-- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX}" +echo "${PYTHON} setup.py install ${SKBUILD_ARGS}" # Workaround for: # DPC++ launched by cmake does not see components of `dpcpp_cpp_rt`, @@ -22,9 +22,9 @@ if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then else WHEELS_BUILD_ARGS="-p manylinux2014_x86_64" fi - ${PYTHON} setup.py ${INSTALL_CMD} bdist_wheel ${WHEELS_BUILD_ARGS} + ${PYTHON} setup.py install bdist_wheel ${WHEELS_BUILD_ARGS} ${SKBUILD_ARGS} cp dist/dpctl*.whl ${WHEELS_OUTPUT_FOLDER} else # Perform regular install - ${PYTHON} setup.py ${INSTALL_CMD} + ${PYTHON} setup.py install ${SKBUILD_ARGS} fi diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index ab01e3c178..c3087c303a 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -18,7 +18,7 @@ requirements: host: - setuptools - cython - - cmake + - cmake >=3.21 - python - make # [unix] - ninja # [win] diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index c858afc7ef..275fe83f80 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -11,6 +11,7 @@ if(WIN32) "-Wunused-function " "-Wuninitialized " "-Wmissing-declarations " + "-Wno-unused-parameter " ) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Ox ${WARNING_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Ox ${WARNING_FLAGS}") @@ -70,11 +71,39 @@ set(DPCTL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) add_custom_target(_build_time_create_dpctl_include ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR} COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/libsyclinterface/include/* ${DPCTL_INCLUDE_DIR}/syclinterface - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/apis/include/* ${DPCTL_INCLUDE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface/Support + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface/Config DEPENDS DPCTLSyclInterface ) +file(GLOB _syclinterface_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/*.h) +foreach(hf ${_syclinterface_h}) + add_custom_command(TARGET _build_time_create_dpctl_include + COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface + ) +endforeach() + +file(GLOB _syclinterface_Support_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Support/*.h) +foreach(hf ${_syclinterface_Support_h}) + add_custom_command(TARGET _build_time_create_dpctl_include + COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface/Support + ) +endforeach() + +file(GLOB _syclinterface_Config_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Config/*.h) +foreach(hf ${_syclinterface_Config_h}) + add_custom_command(TARGET _build_time_create_dpctl_include + COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface/Config + ) +endforeach() + +file(GLOB _apis_h ${CMAKE_CURRENT_SOURCE_DIR}/apis/include/*) +foreach(hf ${_apis_h}) + add_custom_command(TARGET _build_time_create_dpctl_include + COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR} + ) +endforeach() + set(CMAKE_INSTALL_RPATH "$ORIGIN") function(build_dpctl_ext _trgt _src _dest) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index 6ec3314c56..fd78f215a2 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10 FATAL_ERROR) +cmake_minimum_required(VERSION 3.10...3.22 FATAL_ERROR) project( "libDPCTLSYCLInterface" @@ -84,6 +84,7 @@ if(WIN32) "-Wunused-function " "-Wuninitialized " "-Wmissing-declarations " + "-Wno-deprecated-declarations " ) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WARNING_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS}") @@ -195,19 +196,6 @@ file(GLOB MAIN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") file(GLOB SUPPORT_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") file(GLOB CONFIG_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") -if (0) -set(ALL_HEADERS) -foreach(HEADER ${MAIN_HEADERS}) - list(APPEND ALL_HEADERS ${HEADER}) -endforeach() -foreach(HEADER ${SUPPORT_HEADERS}) - list(APPEND ALL_HEADERS ${HEADER}) -endforeach() -foreach(HEADER ${CONFIG_HEADERS}) - list(APPEND ALL_HEADERS ${HEADER}) -endforeach() -endif() - set_target_properties(DPCTLSyclInterface PROPERTIES PUBLIC_HEADER "${MAIN_HEADERS}" @@ -225,6 +213,10 @@ install(TARGETS DPCTLSyclInterface LIBRARY DESTINATION ${_lib_destination} + ARCHIVE + DESTINATION ${_lib_destination} + RUNTIME + DESTINATION ${_lib_destination} PUBLIC_HEADER DESTINATION ${_include_destination} ) From f9654563e657d5f928a5bb313864087f4513e362 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 Jan 2022 11:48:29 -0600 Subject: [PATCH 165/229] Set extra_require keyword for setup.py Two variants are provided: "docs" and "coverage". --- setup.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup.py b/setup.py index a63fdba8ed..c660f01180 100644 --- a/setup.py +++ b/setup.py @@ -97,6 +97,17 @@ def _patched_copy_file(src_file, dest_file, hide_listing=True): install_requires=[ "numpy", ], + extra_require={ + "docs": [ + "Cython", + "sphinx", + "sphinx_rtd_theme", + "pydot", + "graphviz", + "sphinxcontrib-programoutput", + ], + "coverage": ["Cython", "pytest", "pytest-cov", "coverage", "tomli"], + }, keywords="dpctl", classifiers=[ "Development Status :: 3 - Alpha", From 1ccf193a7ae01b164d2b1d1dad0f18372c628b83 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 Jan 2022 14:30:42 -0600 Subject: [PATCH 166/229] Added scripts/get_coverage.py Usage: ``` python scripts/gen_coverage.py ``` Builds coverage binaries, runs both ctests and pytest Allowed options can be retrived with `python scripts/gen_coverage.py --help`. Only Linux is supported, dependencies are assumed to be available. --- scripts/gen_coverage.py | 157 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 scripts/gen_coverage.py diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py new file mode 100644 index 0000000000..1ed055f252 --- /dev/null +++ b/scripts/gen_coverage.py @@ -0,0 +1,157 @@ +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + run_pytest=False, + bin_llvm=None, +): + IS_LIN = False + + if "linux" in sys.platform: + IS_LIN = True + elif sys.platform in ["win32", "cygwin"]: + pass + else: + assert False, sys.platform + " not supported" + + if not IS_LIN: + raise RuntimeError( + "This scripts only supports coverage collection on Linux" + ) + setup_dir = os.path.dirname(os.path.dirname(__file__)) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + "--", + "-G", + "Unix Makefiles", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_GENERATE_COVERAGE=ON", + "-DDPCTL_BUILD_CAPI_TESTS=ON", + "-DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=" + setup_dir, + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + ] + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + env = None + if bin_llvm: + env = { + "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), + } + env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + test_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "tests"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) + subprocess.check_call( + ["make", "-C", test_dir, "lcov-genhtml"], cwd=setup_dir + ) + subprocess.check_call( + [ + "pytest", + "-q", + "-ra", + "--disable-warnings", + "--cov-config", + "pyproject.toml", + "--cov", + "dpctl", + "--cov-report", + "term-missing", + "--pyargs", + "dpctl", + "-vv", + ], + cwd=setup_dir, + shell=False, + ) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl and generate coverage" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--not-oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_false", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + driver.add_argument( + "--skip-pytest", + help="Run pytest and collect coverage", + dest="run_pytest", + action="store_false", + ) + driver.add_argument( + "--bin-llvm", help="Path to folder where llvm-cov can be found" + ) + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" + args.compiler_root = None + icx_path = subprocess.check_output(["which", "icx"]) + bin_dir = os.path.dirname(os.path.dirname(icx_path)) + args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + "bin_llvm", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + run_pytest=args.run_pytest, + bin_llvm=args.bin_llvm, + ) From c7529c6ce4affd5680b92230290673867897a11f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 Jan 2022 14:56:13 -0600 Subject: [PATCH 167/229] Added script to drive setup.py develop for use of OS DPC++ sycl bundle Usage: ``` python scripts/build_sycl_nightly.py --c-compiler=$(which clang) --cxx-compiler=$(which clang++) --compiler-root=$(dirname $(dirname $(which clang))) ``` Option `--cmake-executable` can be used to use newer cmake --- scripts/build_sycl_nightly.py | 115 ++++++++++++++++++++++++++++++++++ scripts/environment.yml | 5 -- 2 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 scripts/build_sycl_nightly.py delete mode 100644 scripts/environment.yml diff --git a/scripts/build_sycl_nightly.py b/scripts/build_sycl_nightly.py new file mode 100644 index 0000000000..5646c980bf --- /dev/null +++ b/scripts/build_sycl_nightly.py @@ -0,0 +1,115 @@ +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + cmake_executable=None, +): + IS_LIN = False + + if "linux" in sys.platform: + IS_LIN = True + elif sys.platform in ["win32", "cygwin"]: + pass + else: + assert False, sys.platform + " not supported" + + if not IS_LIN: + raise RuntimeError( + "This scripts only supports coverage collection on Linux" + ) + setup_dir = os.path.dirname(os.path.dirname(__file__)) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + ] + if cmake_executable: + cmake_args += [ + "--cmake-executable=" + cmake_executable, + ] + cmake_args += [ + "--", + "-G", + "Unix Makefiles", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + ] + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + subprocess.check_call( + cmake_args, shell=False, cwd=setup_dir, env=os.environ + ) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl for in-place installation" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_true", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--cmake-executable", type=str, help="Path to cmake executable" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" + args.compiler_root = None + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + cmake_executable=args.cmake_executable, + ) diff --git a/scripts/environment.yml b/scripts/environment.yml deleted file mode 100644 index 0a063c0a95..0000000000 --- a/scripts/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -channels: - - defaults -dependencies: - - python=3.7 - - conda-build From 51b41b834bfbba09a750aeae92d03fe48a18d5b5 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 27 Jan 2022 06:26:21 -0600 Subject: [PATCH 168/229] Updated quick start guide to reflect changes in setup --- docs/docfiles/user_guides/QuickStart.rst | 38 ++++++++++++++++-------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index 37d6097e4b..fa967a2004 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -50,7 +50,7 @@ Dpctl can also be istalled from Pypi. .. code-block:: bash - python -m pip install --index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple dpctl + python -m pip install --index-url https://pypi.anaconda.org/intel/simple dpctl .. note:: @@ -133,33 +133,41 @@ After building the conda package you may install it by executing: 3.18 instead. -Build and install with setuptools -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Build and install with scikit-build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To build using Python ``setuptools``, the following packages should be +To build using Python ``setuptools`` and ``scikit-build``, the following Python packages should be installed: - ``cython`` - ``numpy`` - ``cmake`` + - ``scikit-build`` - ``ninja`` (only on Windows) - ``gtest`` (optional to run C API tests) + - ``gmock`` (optional to run C API tests) - ``pytest`` (optional to run Python API tests) -Once the prerequisites are installed, building using ``setuptools`` involves The -usual steps +Once the prerequisites are installed, building using ``scikit-build`` involves the usual steps, to build and install: + +.. code-block:: bash -to build and install + python setup.py install -- -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON + +, and to develop: .. code-block:: bash - python setup.py install + python setup.py develop -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON + +On Windows, use ``icx`` for both C and CXX compilers, and use :code:`-G Ninja` for cmake generator. -, and to develop. +Developing on Linux can also be done using driver script: .. code-block:: bash - python setup.py develop + python scripts/build_sycl_nightly.py --oneapi + Building using custom dpcpp --------------------------- @@ -168,12 +176,18 @@ It is possible to build dpctl from source using .. _DPC++ toolchain: https://git instead of the DPC++ compiler that comes with oneAPI. One reason for doing this may be to enable support for CUDA devices. -Following steps in `Build and install with setuptools`_ use command line +Following steps in `Build and install with scikit-build`_ use command line option :code:`--sycl-compiler-prefix`, for example: .. code-block:: bash - python setup.py develop --sycl-compiler-prefix=${DPCPP_ROOT}/llvm/build + python setup.py develop -- -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ONE -DDPCTL_DPCPP_HOME_DIR=${DPCPP_ROOT}/llvm/build -DDPCTL_DPCPP_FROM_ONEAPI=OFF + +Alterantively, the driver script can be used + +.. code-block:: bash + + python scripts/build_sycl_nightly.py --c-compiler=clang --cxx-compiler=clang++ --compiler-root=${DPCPP_ROOT}/llvm/build Available options and their descriptions can be retrieved using option :code:`--help`. From 91705105078f5c6fd71c9bff938ba910634ba252 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 27 Jan 2022 13:05:56 -0600 Subject: [PATCH 169/229] Fixed driver scripts to use abspath for sanitation Removed scripts/build_for_develop* as obsolete. Renamed scripts/build_sycl_nightly.py as scripts/build_locally.py Updated quick start guide to account for the renaming. --- docs/docfiles/user_guides/QuickStart.rst | 7 +- scripts/build_for_develop.bat | 66 ------------------- scripts/build_for_develop.sh | 6 -- ...build_sycl_nightly.py => build_locally.py} | 2 +- scripts/gen_coverage.py | 2 +- 5 files changed, 5 insertions(+), 78 deletions(-) delete mode 100644 scripts/build_for_develop.bat delete mode 100755 scripts/build_for_develop.sh rename scripts/{build_sycl_nightly.py => build_locally.py} (97%) diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index fa967a2004..8c58acd15a 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -166,7 +166,7 @@ Developing on Linux can also be done using driver script: .. code-block:: bash - python scripts/build_sycl_nightly.py --oneapi + python scripts/build_locally.py --oneapi Building using custom dpcpp @@ -176,8 +176,7 @@ It is possible to build dpctl from source using .. _DPC++ toolchain: https://git instead of the DPC++ compiler that comes with oneAPI. One reason for doing this may be to enable support for CUDA devices. -Following steps in `Build and install with scikit-build`_ use command line -option :code:`--sycl-compiler-prefix`, for example: +Following steps in `Build and install with scikit-build`_ use command line option to set relevant cmake variables, for example: .. code-block:: bash @@ -187,7 +186,7 @@ Alterantively, the driver script can be used .. code-block:: bash - python scripts/build_sycl_nightly.py --c-compiler=clang --cxx-compiler=clang++ --compiler-root=${DPCPP_ROOT}/llvm/build + python scripts/build_locally.py --c-compiler=clang --cxx-compiler=clang++ --compiler-root=${DPCPP_ROOT}/llvm/build Available options and their descriptions can be retrieved using option :code:`--help`. diff --git a/scripts/build_for_develop.bat b/scripts/build_for_develop.bat deleted file mode 100644 index 0273a9d62a..0000000000 --- a/scripts/build_for_develop.bat +++ /dev/null @@ -1,66 +0,0 @@ -REM check if oneAPI has been activated, only try activating if not -dpcpp.exe --version >nul 2>&1 -IF %ERRORLEVEL% NEQ 0 ( - set ERRORLEVEL= - call "%ONEAPI_ROOT%\compiler\latest\env\vars.bat" - IF ERRORLEVEL 1 exit /b 1 -) -REM conda uses %ERRORLEVEL% but FPGA scripts can set it. So it should be reseted. -set ERRORLEVEL= - -rmdir /S /Q build_cmake -mkdir build_cmake - -rmdir /S /Q install -mkdir install -cd install -set "INSTALL_PREFIX=%cd%" - -cd ..\build_cmake - -set "DPCPP_ROOT=%ONEAPI_ROOT%\compiler\latest\windows" - -if defined USE_GTEST ( - set "_BUILD_CAPI_TEST=ON" -) else ( - set "_BUILD_CAPI_TEST=OFF" -) -cmake -G Ninja ^ - -DCMAKE_BUILD_TYPE=Release ^ - "-DCMAKE_CXX_FLAGS=-Wno-unused-function /EHa" ^ - "-DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX%" ^ - "-DCMAKE_PREFIX_PATH=%INSTALL_PREFIX%" ^ - "-DDPCPP_ROOT=%DPCPP_ROOT%" ^ - "-DCMAKE_C_COMPILER:PATH=%DPCPP_ROOT%\bin\icx.exe" ^ - "-DCMAKE_CXX_COMPILER:PATH=%DPCPP_ROOT%\bin\dpcpp.exe" ^ - "-DBUILD_CAPI_TESTS=%_BUILD_CAPI_TEST%" ^ - "%cd%\..\dpctl-capi" -IF %ERRORLEVEL% NEQ 0 exit /b 1 - -ninja -n -IF %ERRORLEVEL% NEQ 0 exit /b 1 -if defined USE_GTEST ( - ninja check - IF %ERRORLEVEL% NEQ 0 exit /b 1 -) -ninja install -IF %ERRORLEVEL% NEQ 0 exit /b 1 - -cd .. -xcopy install\lib\*.lib dpctl /E /Y -xcopy install\bin\*.dll dpctl /E /Y - -mkdir dpctl\include -xcopy dpctl-capi\include dpctl\include /E /Y - - -REM required by _sycl_core(dpctl) -set "DPCTL_SYCL_INTERFACE_LIBDIR=dpctl" -set "DPCTL_SYCL_INTERFACE_INCLDIR=dpctl\include" -set "CC=icx.exe" -set "CXX=dpcpp.exe" - -python setup.py clean --all -python setup.py build_ext --inplace develop -python -m unittest dpctl.tests -IF %ERRORLEVEL% NEQ 0 exit /b 1 diff --git a/scripts/build_for_develop.sh b/scripts/build_for_develop.sh deleted file mode 100755 index 530c2a2991..0000000000 --- a/scripts/build_for_develop.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set +xe - -python setup.py clean --all -python setup.py develop --coverage=True -pytest -q -ra --disable-warnings --cov dpctl --cov-report term-missing --pyargs dpctl -vv diff --git a/scripts/build_sycl_nightly.py b/scripts/build_locally.py similarity index 97% rename from scripts/build_sycl_nightly.py rename to scripts/build_locally.py index 5646c980bf..049079cc37 100644 --- a/scripts/build_sycl_nightly.py +++ b/scripts/build_locally.py @@ -24,7 +24,7 @@ def run( raise RuntimeError( "This scripts only supports coverage collection on Linux" ) - setup_dir = os.path.dirname(os.path.dirname(__file__)) + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cmake_args = [ sys.executable, "setup.py", diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 1ed055f252..68ad75ea52 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -25,7 +25,7 @@ def run( raise RuntimeError( "This scripts only supports coverage collection on Linux" ) - setup_dir = os.path.dirname(os.path.dirname(__file__)) + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cmake_args = [ sys.executable, "setup.py", From 955b6a2642d3685926271395c9da7f78772b2de2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 30 Jan 2022 13:12:07 -0600 Subject: [PATCH 170/229] Use stable black, use python 3.9 --- .github/workflows/python_style_checks.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_style_checks.yml b/.github/workflows/python_style_checks.yml index 5e4b9a1d9f..92ae080f53 100644 --- a/.github/workflows/python_style_checks.yml +++ b/.github/workflows/python_style_checks.yml @@ -25,6 +25,10 @@ jobs: # The type of runner that the job will run on runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9] + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -33,16 +37,17 @@ jobs: - uses: actions/setup-python@v2 # Run black code formatter - - uses: psf/black@21.4b2 + - uses: psf/black@stable with: - args: ". --check" + src: "." + options: "--check" flake8: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.9] steps: - uses: actions/checkout@v2 From cb9aa1349f9c0af4b72779e800f5da45850fc10e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 30 Jan 2022 17:46:08 -0600 Subject: [PATCH 171/229] Updated .pre-commit-config to use black 22.1.0 Modified dpctl/tests/test_sycl_event.py per black's whim --- .pre-commit-config.yaml | 2 +- dpctl/tests/test_sycl_event.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54286f0c65..7311c9eb67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 21.10b0 + rev: 22.1.0 hooks: - id: black exclude: "versioneer.py|dpctl/_version.py" diff --git a/dpctl/tests/test_sycl_event.py b/dpctl/tests/test_sycl_event.py index e56337b7ff..7691f00aef 100644 --- a/dpctl/tests/test_sycl_event.py +++ b/dpctl/tests/test_sycl_event.py @@ -202,7 +202,7 @@ def test_sycl_timer(): # device task m1.copy_from_device(m2) # host task - [x ** 2 for x in range(1024)] + [x**2 for x in range(1024)] host_dt, device_dt = timer.dt assert host_dt > device_dt q_no_profiling = dpctl.SyclQueue() From 7dd62c1510f1eadafe7b9510a876206d6c3cb352 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 30 Jan 2022 17:49:11 -0600 Subject: [PATCH 172/229] Submit to black on files from example/ --- examples/cython/sycl_buffer/bench.py | 2 +- examples/cython/sycl_direct_linkage/bench.py | 2 +- examples/cython/usm_memory/run.py | 2 +- examples/pybind11/use_dpctl_syclqueue/example.py | 2 +- examples/python/usm_memory_host_access.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/cython/sycl_buffer/bench.py b/examples/cython/sycl_buffer/bench.py index 9fc493fd18..f8edc91995 100644 --- a/examples/cython/sycl_buffer/bench.py +++ b/examples/cython/sycl_buffer/bench.py @@ -21,7 +21,7 @@ import dpctl -X = np.full((10 ** 4, 4098), 1e-4, dtype="d") +X = np.full((10**4, 4098), 1e-4, dtype="d") # warm-up print("=" * 10 + " Executing warm-up " + "=" * 10) diff --git a/examples/cython/sycl_direct_linkage/bench.py b/examples/cython/sycl_direct_linkage/bench.py index fdff5589ce..bc713a23d7 100644 --- a/examples/cython/sycl_direct_linkage/bench.py +++ b/examples/cython/sycl_direct_linkage/bench.py @@ -19,7 +19,7 @@ import numpy as np import syclbuffer_naive as sb -X = np.full((10 ** 4, 4098), 1e-4, dtype="d") +X = np.full((10**4, 4098), 1e-4, dtype="d") # warm-up print("=" * 10 + " Executing warm-up " + "=" * 10) diff --git a/examples/cython/usm_memory/run.py b/examples/cython/usm_memory/run.py index 720c580250..fa71bba36c 100644 --- a/examples/cython/usm_memory/run.py +++ b/examples/cython/usm_memory/run.py @@ -61,7 +61,7 @@ def gen_option_params( np.allclose(Xgpu, X_ref, atol=1e-5), ) -n_opts = 3 * 10 ** 6 +n_opts = 3 * 10**6 # compute on CPU sycl device diff --git a/examples/pybind11/use_dpctl_syclqueue/example.py b/examples/pybind11/use_dpctl_syclqueue/example.py index 1980ca82d3..7189adecb8 100644 --- a/examples/pybind11/use_dpctl_syclqueue/example.py +++ b/examples/pybind11/use_dpctl_syclqueue/example.py @@ -36,7 +36,7 @@ print("") print("Computing modular reduction using SYCL on a NumPy array") -X = np.random.randint(low=1, high=2 ** 16 - 1, size=10 ** 6, dtype=np.longlong) +X = np.random.randint(low=1, high=2**16 - 1, size=10**6, dtype=np.longlong) modulus_p = 347 Y = eg.offloaded_array_mod( diff --git a/examples/python/usm_memory_host_access.py b/examples/python/usm_memory_host_access.py index 984b4273b5..5fdd24ffc4 100644 --- a/examples/python/usm_memory_host_access.py +++ b/examples/python/usm_memory_host_access.py @@ -34,7 +34,7 @@ # populate buffer from host one byte at a type for i in range(len(ms)): ir = i % 256 - msv[i] = ir ** 2 % 256 + msv[i] = ir**2 % 256 mh = dpmem.MemoryUSMHost(64) mhv = memoryview(mh) From 148e47e3a37169b23bc6ebe0a95a55b2df7cac2a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 16:55:30 -0600 Subject: [PATCH 173/229] test_dpcpp_version is made non-failing (#755) If `mkl_ver >= dpcpp_ver` is detected, the test is xfailed, rather than failed. --- dpctl/tests/test_service.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index 754ba2b09d..cc0638e8bd 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -24,6 +24,8 @@ import os.path import re +import pytest + import dpctl @@ -67,6 +69,11 @@ def test_get_include(): def test_get_dpcppversion(): + """Intent of this test is to verify that libraries from dpcpp_cpp_rt + conda package used at run-time are not from an older oneAPI. Since these + libraries currently do not report the version, this test was using + a proxy (version of Intel(R) Math Kernel Library). + """ incl_dir = dpctl.get_include() libs = glob.glob(os.path.join(incl_dir, "..", "*DPCTLSyclInterface*")) libs = sorted(libs) @@ -80,7 +87,12 @@ def test_get_dpcppversion(): dpcpp_ver = dpcpp_ver.decode("utf-8") mkl_ver = _get_mkl_version_if_present() if mkl_ver is not None: - assert mkl_ver >= dpcpp_ver + if not mkl_ver >= dpcpp_ver: + pytest.xfail( + reason="Flaky test: Investigate Math Kernel Library " + f"library version {mkl_ver} being older than " + f"DPC++ version {dpcpp_ver} used to build dpctl" + ) def test___version__(): From d1a8bf20e37f34cdb423a5e3f027fbb29558644c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 28 Jan 2022 06:46:24 -0600 Subject: [PATCH 174/229] CYTHON_TRACE must be set when generating coverage Setting of this proprocessor variable went missing in the move to cmake + scikit-build, resulting in coverage not going through Cython files anymore. This change should restore that, and thus improve the coverage percentage. --- dpctl/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 275fe83f80..1d730e9933 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -111,6 +111,10 @@ function(build_dpctl_ext _trgt _src _dest) add_library(${_trgt} MODULE ${_generated_src}) target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) add_dependencies(${_trgt} _build_time_create_dpctl_include) + if (DPCTL_GENERATE_COVERAGE) + target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_PROFILE=1) + target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer) + endif() target_link_libraries(${_trgt} DPCTLSyclInterface) target_link_options(${_trgt} PRIVATE "LINKER:${DPCTL_LDFLAGS}") python_extension_module(${_trgt}) From b124c9474e5418d99bb0970adbd29a394cf5d687 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 28 Jan 2022 18:16:06 -0600 Subject: [PATCH 175/229] Renamed copy_generated_headers.cmake -> copy_existing.cmake --- dpctl/CMakeLists.txt | 6 +++--- .../{copy_generated_headers.cmake => copy_existing.cmake} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename dpctl/cmake/{copy_generated_headers.cmake => copy_existing.cmake} (100%) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 1d730e9933..9ab20afbf3 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -112,7 +112,7 @@ function(build_dpctl_ext _trgt _src _dest) target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) add_dependencies(${_trgt} _build_time_create_dpctl_include) if (DPCTL_GENERATE_COVERAGE) - target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_PROFILE=1) + target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1) target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer) endif() target_link_libraries(${_trgt} DPCTLSyclInterface) @@ -128,11 +128,11 @@ function(build_dpctl_ext _trgt _src _dest) COMMAND ${CMAKE_COMMAND} -DSOURCE_FILE=${_generated_public_h} -DDEST=${CMAKE_CURRENT_SOURCE_DIR} - -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_generated_headers.cmake + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake COMMAND ${CMAKE_COMMAND} -DSOURCE_FILE=${_generated_api_h} -DDEST=${CMAKE_CURRENT_SOURCE_DIR} - -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_generated_headers.cmake + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake DEPENDS ${_trgt} VERBATIM COMMENT "Copying Cython-generated headers to destination" diff --git a/dpctl/cmake/copy_generated_headers.cmake b/dpctl/cmake/copy_existing.cmake similarity index 100% rename from dpctl/cmake/copy_generated_headers.cmake rename to dpctl/cmake/copy_existing.cmake From ccacd4af9cf18fa80d989f82e54b2d1feba75966 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 30 Jan 2022 12:15:52 -0600 Subject: [PATCH 176/229] Changes for Cython.Coverage to work with scikit-build When cython converts .pyx files to cxx it is now provided a working directory option. This makes sure that generated `__pyx_f` array lists sources relative to project directory. When generating coverage, cmake adds a target to copy generated .cxx files to the location of .pyx file. --- dpctl/CMakeLists.txt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 9ab20afbf3..69292f897e 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -1,6 +1,8 @@ find_package(PythonExtensions REQUIRED) find_package(NumPy REQUIRED) + +set(CYTHON_FLAGS "-w ${CMAKE_SOURCE_DIR}") find_package(Cython REQUIRED) if(WIN32) @@ -135,8 +137,21 @@ function(build_dpctl_ext _trgt _src _dest) -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake DEPENDS ${_trgt} VERBATIM - COMMENT "Copying Cython-generated headers to destination" + COMMENT "Copying Cython-generated headers to dpctl" ) + if (DPCTL_GENERATE_COVERAGE) + set(_copy_cxx_trgt "${_trgt}_copy_cxx") + add_custom_target( + ${_copy_cxx_trgt} ALL + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_src} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake + DEPENDS ${_trgt} + VERBATIM + COMMENT "Copying Cython-generated source to dpctl" + ) + endif() install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest}) endfunction() From 6fab7027ea9eba6459774224098b2297885b4c44 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 30 Jan 2022 13:08:12 -0600 Subject: [PATCH 177/229] Added *.cxx to dpctl/.gitignore file --- dpctl/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/dpctl/.gitignore b/dpctl/.gitignore index 0c03f1ed6a..3e23a8af25 100644 --- a/dpctl/.gitignore +++ b/dpctl/.gitignore @@ -1,5 +1,6 @@ *.so *.cpp +*.cxx *.c *.h memory/*.h From e1189e509ec1c6510f20e40d4c126cd30af9344d Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 26 Jan 2022 18:55:26 -0600 Subject: [PATCH 178/229] SyclEvent deleter no longer waits Added _host_task_utils.hpp file that provides function to submit host task to decrement reference count for each Python object passed to one gated by passed event references. Host task is submitted to sycl::queue behind the given QRef. SyclQueue.submit changes the mechanism by which it ensures that Python objects (and memory they hold) that the kernel works on are not GC collected before the kernel completes its execution. We used to store reference to arguments in the SyclEvent object returned by SyclQueue.submit. To avoid GC-ing kernel arguments before kernel completes execution, deleter of SyclEvent has to call `SyclEvent.wait`. With this PR `SyclQueue.submit` increments reference counts of arguments after submission, then schedules a host task on the same queue, dependent on the kernel event, which aquires GIL and decrements reference counts. After this change the SyclEven.__dealloc__ no longer has to wait. It also no longer needs to store args attribute. This is now deperecated and will be removed. --- dpctl/CMakeLists.txt | 2 ++ dpctl/_host_task_util.hpp | 42 +++++++++++++++++++++++++++++++++++++++ dpctl/_sycl_event.pyx | 2 +- dpctl/_sycl_queue.pyx | 24 +++++++++++++++++++--- 4 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 dpctl/_host_task_util.hpp diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 69292f897e..3ca3c9a79d 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -161,6 +161,8 @@ foreach(_cy_file ${_cython_sources}) build_dpctl_ext(${_trgt} ${_cy_file} "dpctl") endforeach() +target_include_directories(_sycl_queue PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + add_subdirectory(program) add_subdirectory(memory) add_subdirectory(tensor) diff --git a/dpctl/_host_task_util.hpp b/dpctl/_host_task_util.hpp new file mode 100644 index 0000000000..9671ed1547 --- /dev/null +++ b/dpctl/_host_task_util.hpp @@ -0,0 +1,42 @@ +#include "Python.h" +#include "syclinterface/dpctl_data_types.h" +#include + +int async_dec_ref(DPCTLSyclQueueRef QRef, + PyObject **obj_array, + size_t obj_array_size, + DPCTLSyclEventRef *ERefs, + size_t nERefs) +{ + + sycl::queue *q = reinterpret_cast(QRef); + + std::vector obj_vec; + obj_vec.reserve(obj_array_size); + for (size_t obj_id = 0; obj_id < obj_array_size; ++obj_id) { + obj_vec.push_back(obj_array[obj_id]); + } + + try { + q->submit([&](sycl::handler &cgh) { + for (size_t ev_id = 0; ev_id < nERefs; ++ev_id) { + cgh.depends_on( + *(reinterpret_cast(ERefs[ev_id]))); + } + cgh.host_task([obj_array_size, obj_vec]() { + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + for (size_t i = 0; i < obj_array_size; ++i) { + Py_DECREF(obj_vec[i]); + } + PyGILState_Release(gstate); + } + }); + }); + } catch (const std::exception &e) { + return 1; + } + + return 0; +} diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index bd680eadf7..77f3e7a0f0 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -98,7 +98,7 @@ cdef class _SyclEvent: def __dealloc__(self): if (self._event_ref): - with nogil: DPCTLEvent_Wait(self._event_ref) + # with nogil: DPCTLEvent_Wait(self._event_ref) DPCTLEvent_Delete(self._event_ref) self._event_ref = NULL self.args = None diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 1e1b15813b..896c568061 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -65,11 +65,17 @@ import ctypes from .enum_types import backend_type from cpython cimport pycapsule +from cpython.ref cimport Py_INCREF, PyObject from libc.stdlib cimport free, malloc import collections.abc import logging + +cdef extern from "_host_task_util.hpp": + int async_dec_ref(DPCTLSyclQueueRef, PyObject **, size_t, DPCTLSyclEventRef *, size_t) nogil + + __all__ = [ "SyclQueue", "SyclKernelInvalidRangeError", @@ -714,12 +720,14 @@ cdef class SyclQueue(_SyclQueue): cdef _arg_data_type *kargty = NULL cdef DPCTLSyclEventRef *depEvents = NULL cdef DPCTLSyclEventRef Eref = NULL - cdef int ret + cdef int ret = 0 cdef size_t gRange[3] cdef size_t lRange[3] cdef size_t nGS = len(gS) cdef size_t nLS = len(lS) if lS is not None else 0 cdef size_t nDE = len(dEvents) if dEvents is not None else 0 + cdef PyObject **arg_objects = NULL + cdef ssize_t i = 0 # Allocate the arrays to be sent to DPCTLQueue_Submit kargs = malloc(len(args) * sizeof(void*)) @@ -820,8 +828,18 @@ cdef class SyclQueue(_SyclQueue): raise SyclKernelSubmitError( "Kernel submission to Sycl queue failed." ) - - return SyclEvent._create(Eref, args) + # increment reference counts to each argument + arg_objects = malloc(len(args) * sizeof(PyObject *)) + for i in range(len(args)): + arg_objects[i] = (args[i]) + Py_INCREF( arg_objects[i]) + + # schedule decrement + async_dec_ref(self.get_queue_ref(), arg_objects, len(args), &Eref, 1) + # free memory + free(arg_objects) + + return SyclEvent._create(Eref, []) cpdef void wait(self): with nogil: DPCTLQueue_Wait(self._queue_ref) From 68616129775d1753d21ddf5e938ee763d80d236b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 28 Jan 2022 18:32:19 -0600 Subject: [PATCH 179/229] added license header and explanatory comment --- dpctl/_host_task_util.hpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dpctl/_host_task_util.hpp b/dpctl/_host_task_util.hpp index 9671ed1547..5208e11d2c 100644 --- a/dpctl/_host_task_util.hpp +++ b/dpctl/_host_task_util.hpp @@ -1,3 +1,34 @@ +//===--- _host_tasl_util.hpp - Implements async DECREF =// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements a utility function to schedule host task to a sycl +/// queue depending on given array of sycl events to decrement reference counts +/// for the given array of Python objects. +/// +/// N.B.: The host task attempts to acquire GIL, so queue wait, event wait and +/// other synchronization mechanisms should be called after releasing the GIL to +/// avoid deadlocks. +/// +//===----------------------------------------------------------------------===// + #include "Python.h" #include "syclinterface/dpctl_data_types.h" #include From f60faceca1fed0a90fd060fcfa2db66ce47897ae Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 09:47:09 -0600 Subject: [PATCH 180/229] removed commented out code line --- dpctl/_sycl_event.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index 77f3e7a0f0..c64478d3c6 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -98,7 +98,6 @@ cdef class _SyclEvent: def __dealloc__(self): if (self._event_ref): - # with nogil: DPCTLEvent_Wait(self._event_ref) DPCTLEvent_Delete(self._event_ref) self._event_ref = NULL self.args = None From fad31cce53e7d012929b3553702b3c30cb9e5308 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 09:48:16 -0600 Subject: [PATCH 181/229] Handle error in async_dec_ref: decrement ref counts and wait for the job to complete --- dpctl/_sycl_queue.pyx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 896c568061..30a64fec4c 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -65,7 +65,7 @@ import ctypes from .enum_types import backend_type from cpython cimport pycapsule -from cpython.ref cimport Py_INCREF, PyObject +from cpython.ref cimport Py_DECREF, Py_INCREF, PyObject from libc.stdlib cimport free, malloc import collections.abc @@ -835,7 +835,13 @@ cdef class SyclQueue(_SyclQueue): Py_INCREF( arg_objects[i]) # schedule decrement - async_dec_ref(self.get_queue_ref(), arg_objects, len(args), &Eref, 1) + if async_dec_ref(self.get_queue_ref(), arg_objects, len(args), &Eref, 1): + # async task submission failed, decrement ref counts and wait + for i in range(len(args)): + arg_objects[i] = (args[i]) + Py_DECREF( arg_objects[i]) + with nogil: DPCTLEvent_Wait(Eref) + # free memory free(arg_objects) From 8effe7762ea76d02d8db82426f568410ea207ffe Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 09:49:41 -0600 Subject: [PATCH 182/229] Add test to check asynchronicity of submit --- dpctl/tests/test_sycl_kernel_submit.py | 98 +++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/dpctl/tests/test_sycl_kernel_submit.py b/dpctl/tests/test_sycl_kernel_submit.py index 3f3a03bb97..8a60294dd5 100644 --- a/dpctl/tests/test_sycl_kernel_submit.py +++ b/dpctl/tests/test_sycl_kernel_submit.py @@ -25,6 +25,7 @@ import dpctl import dpctl.memory as dpctl_mem import dpctl.program as dpctl_prog +import dpctl.tensor as dpt @pytest.mark.parametrize( @@ -107,4 +108,99 @@ def test_create_program_from_source(ctype_str, dtype, ctypes_ctor): ref_c = a * np.array(d, dtype=dtype) + b host_dt, device_dt = timer.dt assert type(host_dt) is float and type(device_dt) is float - assert np.allclose(c, ref_c), "Faled for {}, {}".formatg(r, lr) + assert np.allclose(c, ref_c), "Failed for {}, {}".formatg(r, lr) + + +def test_async_submit(): + try: + q = dpctl.SyclQueue("opencl") + except dpctl.SyclQueueCreationError: + pytest.skip("OpenCL queue could not be created") + oclSrc = ( + "kernel void kern1(global unsigned int *res, unsigned int mod) {" + " size_t index = get_global_id(0);" + " int ri = (index % mod);" + " res[index] = (ri * ri) % mod;" + "}" + " " + "kernel void kern2(global unsigned int *res, unsigned int mod) {" + " size_t index = get_global_id(0);" + " int ri = (index % mod);" + " int ri2 = (ri * ri) % mod;" + " res[index] = (ri2 * ri) % mod;" + "}" + " " + "kernel void kern3(" + " global unsigned int *res, global unsigned int *arg1, " + " global unsigned int *arg2)" + "{" + " size_t index = get_global_id(0);" + " res[index] = " + " (arg1[index] < arg2[index]) ? arg1[index] : arg2[index];" + "}" + ) + prog = dpctl_prog.create_program_from_source(q, oclSrc) + kern1Kernel = prog.get_sycl_kernel("kern1") + kern2Kernel = prog.get_sycl_kernel("kern2") + kern3Kernel = prog.get_sycl_kernel("kern3") + + assert isinstance(kern1Kernel, dpctl_prog.SyclKernel) + assert isinstance(kern2Kernel, dpctl_prog.SyclKernel) + assert isinstance(kern2Kernel, dpctl_prog.SyclKernel) + + n = 1024 * 1024 + X = dpt.empty((3, n), dtype="u4", usm_type="device", sycl_queue=q) + first_row = dpctl_mem.as_usm_memory(X[0]) + second_row = dpctl_mem.as_usm_memory(X[1]) + third_row = dpctl_mem.as_usm_memory(X[2]) + + e1 = q.submit( + kern1Kernel, + [ + first_row, + ctypes.c_uint(17), + ], + [ + n, + ], + ) + e2 = q.submit( + kern2Kernel, + [ + second_row, + ctypes.c_uint(27), + ], + [ + n, + ], + ) + e3 = q.submit( + kern3Kernel, + [third_row, first_row, second_row], + [ + n, + ], + None, + [e1, e2], + ) + status_complete = dpctl.event_status_type.complete + assert not all( + [ + e == status_complete + for e in ( + e1.execution_status, + e2.execution_status, + e3.execution_status, + ) + ] + ) + + e3.wait() + Xnp = dpt.asnumpy(X) + Xref = np.empty((3, n), dtype="u4") + for i in range(n): + Xref[0, i] = (i * i) % 17 + Xref[1, i] = (i * i * i) % 27 + Xref[2, i] = min(Xref[0, i], Xref[1, i]) + + assert np.array_equal(Xnp, Xref) From 9bc62ab9a561a62e77127fc1db15675d6567d9e1 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Sat, 18 Dec 2021 02:18:32 +0300 Subject: [PATCH 183/229] Implement InitLogger and ShutdownLogger funcs for glog and change error_handlers --- .../helper/source/dpctl_error_handlers.cpp | 46 ++++++++++++++++--- libsyclinterface/include/dpctl_service.h | 4 ++ libsyclinterface/source/dpctl_service.cpp | 38 ++++++++++++++- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp index 3755608cf1..3a5752e55b 100644 --- a/libsyclinterface/helper/source/dpctl_error_handlers.cpp +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -24,7 +24,12 @@ //===----------------------------------------------------------------------===// #include "dpctl_error_handlers.h" +#include "dpctl_service.h" #include +#include +#ifdef ENABLE_GLOG +#include +#endif void DPCTL_AsyncErrorHandler::operator()( const cl::sycl::exception_list &exceptions) @@ -59,6 +64,25 @@ int requested_verbosity_level(void) return requested_level; } + +void output_message(std::string ss_str, [[maybe_unused]] error_level error_type) +{ +#ifdef ENABLE_GLOG + switch (error_type) { + case error_level::error: + LOG(ERROR) << ss_str; + break; + case error_level::warning: + LOG(WARNING) << ss_str; + break; + default: + LOG(FATAL) << ss_str; + } +#else + std::cerr << ss_str; +#endif +} + } // namespace void error_handler(const std::exception &e, @@ -70,9 +94,14 @@ void error_handler(const std::exception &e, int requested_level = requested_verbosity_level(); int error_level = static_cast(error_type); - if (requested_level >= error_level) { - std::cerr << e.what() << " in " << func_name << " at " << file_name - << ":" << line_num << std::endl; + bool to_output = requested_level >= error_level; + + if (to_output) { + std::stringstream ss; + ss << e.what() << " in " << func_name << " at " << file_name << ":" + << line_num << std::endl; + + output_message(ss.str(), error_type); } } @@ -85,8 +114,13 @@ void error_handler(const std::string &what, int requested_level = requested_verbosity_level(); int error_level = static_cast(error_type); - if (requested_level >= error_level) { - std::cerr << what << " In " << func_name << " at " << file_name << ":" - << line_num << std::endl; + bool to_output = requested_level >= error_level; + + if (to_output) { + std::stringstream ss; + ss << what << " in " << func_name << " at " << file_name << ":" + << line_num << std::endl; + + output_message(ss.str(), error_type); } } diff --git a/libsyclinterface/include/dpctl_service.h b/libsyclinterface/include/dpctl_service.h index e66c1a1c47..e77b45b508 100644 --- a/libsyclinterface/include/dpctl_service.h +++ b/libsyclinterface/include/dpctl_service.h @@ -44,3 +44,7 @@ DPCTL_API __dpctl_give const char *DPCTLService_GetDPCPPVersion(void); DPCTL_C_EXTERN_C_END + +void DPCTLService_InitLogger(const char *app_name, const char *log_dir); + +void DPCTLService_ShutdownLogger(void); diff --git a/libsyclinterface/source/dpctl_service.cpp b/libsyclinterface/source/dpctl_service.cpp index 3457980721..79a21082a6 100644 --- a/libsyclinterface/source/dpctl_service.cpp +++ b/libsyclinterface/source/dpctl_service.cpp @@ -26,13 +26,49 @@ #include "dpctl_service.h" #include "Config/dpctl_config.h" -#include "../helper/include/dpctl_string_utils.hpp" +#include "dpctl_string_utils.hpp" #include #include #include +#include +#ifdef ENABLE_GLOG +#include +#endif __dpctl_give const char *DPCTLService_GetDPCPPVersion(void) { std::string version = DPCTL_DPCPP_VERSION; return dpctl::helper::cstring_from_string(version); } + +#ifdef ENABLE_GLOG + +void DPCTLService_InitLogger(const char *app_name, const char *log_dir) +{ + google::InitGoogleLogging(app_name); + google::EnableLogCleaner(0); + google::InstallFailureSignalHandler(); + FLAGS_colorlogtostderr = true; + FLAGS_stderrthreshold = google::FATAL; + struct stat buffer; + if (stat(log_dir, &buffer) != 0) { + FLAGS_log_dir = log_dir; + } + else { + std::cerr << "Directory not found. Log files will be created in the " + "default logging directory.\n"; + } +} + +void DPCTLService_ShutdownLogger(void) +{ + google::ShutdownGoogleLogging(); +} + +#else +void DPCTLService_InitLogger([[maybe_unused]] const char *app_name, + [[maybe_unused]] const char *log_dir){}; + +void DPCTLService_ShutdownLogger(void){}; + +#endif From 593af45e6ee0efde475f3130390b79f78eaeb512 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Sat, 18 Dec 2021 02:33:27 +0300 Subject: [PATCH 184/229] Provide DPCTL_ENABLE_GLOG cmake option to turn on logging --- libsyclinterface/CMakeLists.txt | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index fd78f215a2..a6c73f3878 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -38,6 +38,11 @@ option(DPCTL_BUILD_CAPI_TESTS "Build dpctl C API google tests" OFF ) +# Option to turn on logging support for dpctl C API +option(DPCTL_ENABLE_GLOG + "Enable the Google logging module" + OFF +) # Minimum version requirement only when oneAPI dpcpp is used. if(DPCTL_DPCPP_FROM_ONEAPI) @@ -166,12 +171,25 @@ target_include_directories(DPCTLSyclInterface ${CMAKE_CURRENT_SOURCE_DIR}/helper/include/ ${IntelSycl_SYCL_INCLUDE_DIR} ) - target_link_libraries(DPCTLSyclInterface - PRIVATE ${IntelSycl_SYCL_LIBRARY} - PRIVATE ${IntelSycl_OPENCL_LIBRARY} + PRIVATE ${IntelSycl_SYCL_LIBRARY} + PRIVATE ${IntelSycl_OPENCL_LIBRARY} ) +if(DPCTL_ENABLE_GLOG) + find_package(glog REQUIRED) + + target_include_directories(DPCTLSyclInterface + PRIVATE + glog::glog + ) + target_compile_definitions(DPCTLSyclInterface PRIVATE ENABLE_GLOG) + target_link_libraries(DPCTLSyclInterface + PRIVATE glog::glog + ) +endif() + + include(GetProjectVersion) # the get_version function is defined in the GetProjectVersion module and # defines: VERSION, SEMVER, MAJOR, MINOR, PATCH. These variables are populated From 73951d881ef3d4fccd150f7b4c05008173ffbef7 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Sat, 18 Dec 2021 02:28:22 +0300 Subject: [PATCH 185/229] Add glog function argument for build_backend class --- scripts/build_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index d188dfb8b3..ecbcffd2fd 100644 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -19,7 +19,7 @@ def build_backend( - l0_support=False, code_coverage=False, sycl_compiler_prefix=None + l0_support=False, code_coverage=False, glog=False, sycl_compiler_prefix=None ): import glob import os @@ -99,6 +99,8 @@ def build_backend( "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang++"), ] + if glog: + cmake_compiler_args.append("-DDPCTL_ENABLE_GLOG=ON") if code_coverage: cmake_args = ( [ @@ -177,6 +179,8 @@ def build_backend( "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang++.exe"), ] + if glog: + cmake_compiler_args.append("-DDPCTL_ENABLE_GLOG=ON") cmake_args = ( [ "cmake", From 3fd122b48bbd7537cd4444eca83158423fd2fd4a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 27 Jan 2022 15:14:12 -0600 Subject: [PATCH 186/229] scripts/build_locally.py supports --glog --- scripts/build_locally.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 049079cc37..21a54139a0 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -10,6 +10,7 @@ def run( level_zero=True, compiler_root=None, cmake_executable=None, + use_glog=False, ): IS_LIN = False @@ -43,6 +44,7 @@ def run( "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + "-DDPCTL_ENABLE_GLOG:BOOL=" + ("ON" if use_glog else "OFF"), ] if compiler_root: cmake_args += [ @@ -82,6 +84,12 @@ def run( dest="level_zero", action="store_false", ) + driver.add_argument( + "--glog", + help="DPCTLSyclInterface uses Google logger", + dest="glog", + action="store_true", + ) args = parser.parse_args() if args.oneapi: @@ -112,4 +120,5 @@ def run( level_zero=args.level_zero, compiler_root=args.compiler_root, cmake_executable=args.cmake_executable, + use_glog=args.glog, ) From 971d20a55b9eac33361f3e1c9983a31da226119f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 13:33:24 -0600 Subject: [PATCH 187/229] Fixes for DPCTLService_InitLogger Replaced use of `stat` function with use of `filesystem::is_directory`. Stat was always erroring out even for path to existing directory. Created `dpctl._dev` module for developers. It adds ```python dpctl._dev.init_logger(log_dir=None) ``` Function to initialize the logger. It uses `"dpctl"` for application name, and supplied log directory. `log_dir=None` is interpreted as use the current working directory. ```pyton dpctl._dev.fini_logger() ``` Call ShutdownLogger. ```python dpctl._dev.verbose(verbostiy="warning", log_dir=None) ``` Context manager which initializes the logger, and sets the verbosity level for a given block of Python code: Example: ``` In [1]: import dpctl, dpctl._dev as dd In [2]: with dd.verbose(): dpctl.SyclDevice().parent_device No parent for device because it is not a subdevice -33 (CL_INVALID_DEVICE) in DPCTLDevice_GetParentDevice at /localdisk/work/opavlyk/repos/dpctl/libsyclinterface/source/dpctl_sycl_device_interface.cpp:540 In [3]: dpctl.SyclDevice().parent_device # no message In [4]: with dd.verbose(): dpctl.SyclDevice().parent_device No parent for device because it is not a subdevice -33 (CL_INVALID_DEVICE) in DPCTLDevice_GetParentDevice at /localdisk/work/opavlyk/repos/dpctl/libsyclinterface/source/dpctl_sycl_device_interface.cpp:540 In [5]: dpctl.SyclDevice().parent_device # no message In [6]: quit ``` Export DPCTLService_InitLogger and DPCTLService_FiniLogger --- dpctl/_dev.pyx | 77 +++++++++++++++++++++++ libsyclinterface/include/dpctl_service.h | 18 +++++- libsyclinterface/source/dpctl_service.cpp | 10 ++- 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 dpctl/_dev.pyx diff --git a/dpctl/_dev.pyx b/dpctl/_dev.pyx new file mode 100644 index 0000000000..572fa16ec5 --- /dev/null +++ b/dpctl/_dev.pyx @@ -0,0 +1,77 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +""" Implements developer utilities. +""" +import contextlib +import os + + +cdef extern from "syclinterface/dpctl_service.h": + cdef void DPCTLService_InitLogger(const char *, const char *) + cdef void DPCTLService_ShutdownLogger() + + +def init_logger(log_dir=None): + """Initialize logger to use given directory to save logs. + + The call has no effect if `dpctl` was not built to use logger. + """ + cdef bytes p = b"" + cdef const char *app_name = "dpctl" + if log_dir is None: + log_dir = os.getcwd() + if not os.path.exists(log_dir): + raise ValueError(f"Path {log_dir} does not exist") + if isinstance(log_dir, str): + p = bytes(log_dir, "utf-8") + else: + p = bytes(log_dir) + DPCTLService_InitLogger(app_name, p) + + +def fini_logger(): + """Finilize logger. + + The call has no effect if `dpctl` was not built to use logger. + """ + DPCTLService_ShutdownLogger() + + +@contextlib.contextmanager +def verbose(verbosity="warning", log_dir=None): + """Context manager that activate verbosity""" + _allowed_verbosity = ["warning", "error"] + if not verbosity in _allowed_verbosity: + raise ValueError( + f"Verbosity argument not understood. " + f"Permitted values are {_allowed_verbosity}" + ) + init_logger(log_dir=log_dir) + _saved = os.environ.get("DPCTL_VERBOSITY", None) + os.environ["DPCTL_VERBOSITY"] = verbosity + try: + yield + finally: + fini_logger() + if _saved: + os.environ["DPCTL_VERBOSITY"] = _saved + else: + del os.environ["DPCTL_VERBOSITY"] diff --git a/libsyclinterface/include/dpctl_service.h b/libsyclinterface/include/dpctl_service.h index e77b45b508..d7f9cfb552 100644 --- a/libsyclinterface/include/dpctl_service.h +++ b/libsyclinterface/include/dpctl_service.h @@ -43,8 +43,22 @@ DPCTL_C_EXTERN_C_BEGIN DPCTL_API __dpctl_give const char *DPCTLService_GetDPCPPVersion(void); -DPCTL_C_EXTERN_C_END - +/*! + * @brief Initialize logger if compiled to use logger, no-op otherwise. + * + * @param app_name C-string for application name reflected in the log. + * @paral log_dir C-string for directory where log files are placed. + * @ingroup Service + */ +DPCTL_API void DPCTLService_InitLogger(const char *app_name, const char *log_dir); +/*! + * @brief Finilize logger if enabled, no-op otherwise. + * + * @ingroup Service + */ +DPCTL_API void DPCTLService_ShutdownLogger(void); + +DPCTL_C_EXTERN_C_END diff --git a/libsyclinterface/source/dpctl_service.cpp b/libsyclinterface/source/dpctl_service.cpp index 79a21082a6..944786e1a4 100644 --- a/libsyclinterface/source/dpctl_service.cpp +++ b/libsyclinterface/source/dpctl_service.cpp @@ -30,8 +30,8 @@ #include #include #include -#include #ifdef ENABLE_GLOG +#include #include #endif @@ -50,8 +50,12 @@ void DPCTLService_InitLogger(const char *app_name, const char *log_dir) google::InstallFailureSignalHandler(); FLAGS_colorlogtostderr = true; FLAGS_stderrthreshold = google::FATAL; - struct stat buffer; - if (stat(log_dir, &buffer) != 0) { + + namespace fs = std::filesystem; + const fs::path path(log_dir); + std::error_code ec; + + if (fs::is_directory(path, ec)) { FLAGS_log_dir = log_dir; } else { From 2f6aaec29bea46ef8cb560b2dc805489ec057f23 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 13:57:51 -0600 Subject: [PATCH 188/229] Add tests to dpctl._dev.verbose --- .flake8 | 1 + dpctl/tests/test_service.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.flake8 b/.flake8 index c4dc50e407..0256f97cad 100644 --- a/.flake8 +++ b/.flake8 @@ -13,6 +13,7 @@ exclude = .git per-file-ignores = + dpctl/_dev.pyx: E999 dpctl/_sycl_context.pyx: E999, E225, E227 dpctl/_sycl_device.pyx: E999, E225 dpctl/_sycl_device_factory.pyx: E999, E225 diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index cc0638e8bd..3c2fa6bf9f 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -107,3 +107,20 @@ def test___version__(): r"0|[1-9][0-9]*))?(\+.*)?$" ) assert re.match(reg_expr, dpctl_ver) is not None + + +def test_dev_utils(): + import dpctl._dev as dd + + with dd.verbose(): + dpctl.SyclDevice().parent_device + with dd.verbose(verbosity="error"): + dpctl.SyclDevice().parent_device + with pytest.raises(ValueError): + with dd.verbose(verbosity="blah"): + dpctl.SyclDevice().parent_device + with dd.verbose(log_dir="/tmp"): + dpctl.SyclDevice().parent_device + with pytest.raises(ValueError): + with dd.verbose(log_dir="/not_a_dir"): + dpctl.SyclDevice().parent_device From 47c3e9c1ed14f1f9a1fdade1ca61d3695900ad8f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 16:55:35 -0600 Subject: [PATCH 189/229] Renaming to improve self-descriptiveness. 1. GLOG defaults (log_dir=None) to outputing to color-enhanced std::cerr 2. Specifying valid log_dir changes the mode to logging 3. Context manager renamed from verbose to syclinterface_diagnostic Test updated. --- .flake8 | 2 +- dpctl/{_dev.pyx => _diagnostics.pyx} | 41 ++++++++++--------- dpctl/tests/test_service.py | 19 +++++---- .../helper/source/dpctl_error_handlers.cpp | 19 ++++++--- libsyclinterface/source/dpctl_service.cpp | 21 +++++----- 5 files changed, 60 insertions(+), 42 deletions(-) rename dpctl/{_dev.pyx => _diagnostics.pyx} (67%) diff --git a/.flake8 b/.flake8 index 0256f97cad..5a20d20b6e 100644 --- a/.flake8 +++ b/.flake8 @@ -13,7 +13,7 @@ exclude = .git per-file-ignores = - dpctl/_dev.pyx: E999 + dpctl/_diagnostics.pyx: E999 dpctl/_sycl_context.pyx: E999, E225, E227 dpctl/_sycl_device.pyx: E999, E225 dpctl/_sycl_device_factory.pyx: E999, E225 diff --git a/dpctl/_dev.pyx b/dpctl/_diagnostics.pyx similarity index 67% rename from dpctl/_dev.pyx rename to dpctl/_diagnostics.pyx index 572fa16ec5..dc98cb29db 100644 --- a/dpctl/_dev.pyx +++ b/dpctl/_diagnostics.pyx @@ -29,26 +29,27 @@ cdef extern from "syclinterface/dpctl_service.h": cdef void DPCTLService_ShutdownLogger() -def init_logger(log_dir=None): +def _init_logger(log_dir=None): """Initialize logger to use given directory to save logs. The call has no effect if `dpctl` was not built to use logger. """ cdef bytes p = b"" cdef const char *app_name = "dpctl" - if log_dir is None: - log_dir = os.getcwd() - if not os.path.exists(log_dir): - raise ValueError(f"Path {log_dir} does not exist") - if isinstance(log_dir, str): - p = bytes(log_dir, "utf-8") - else: - p = bytes(log_dir) - DPCTLService_InitLogger(app_name, p) + cdef char *ld_cstr = NULL + if log_dir: + if not os.path.exists(log_dir): + raise ValueError(f"Path {log_dir} does not exist") + if isinstance(log_dir, str): + p = bytes(log_dir, "utf-8") + else: + p = bytes(log_dir) + ld_cstr = p + DPCTLService_InitLogger(app_name, ld_cstr) -def fini_logger(): - """Finilize logger. +def _shutdown_logger(): + """Finalize logger. The call has no effect if `dpctl` was not built to use logger. """ @@ -56,22 +57,24 @@ def fini_logger(): @contextlib.contextmanager -def verbose(verbosity="warning", log_dir=None): - """Context manager that activate verbosity""" +def syclinterface_diagnostics(verbosity="warning", log_dir=None): + """Context manager that activate verbosity of DPCTLSyclInterface + function calls. + """ _allowed_verbosity = ["warning", "error"] if not verbosity in _allowed_verbosity: raise ValueError( f"Verbosity argument not understood. " f"Permitted values are {_allowed_verbosity}" ) - init_logger(log_dir=log_dir) - _saved = os.environ.get("DPCTL_VERBOSITY", None) + _init_logger(log_dir=log_dir) + _saved_verbosity = os.environ.get("DPCTL_VERBOSITY", None) os.environ["DPCTL_VERBOSITY"] = verbosity try: yield finally: - fini_logger() - if _saved: - os.environ["DPCTL_VERBOSITY"] = _saved + _shutdown_logger() + if _saved_verbosity: + os.environ["DPCTL_VERBOSITY"] = _saved_verbosity else: del os.environ["DPCTL_VERBOSITY"] diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index 3c2fa6bf9f..53f895d20b 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -110,17 +110,22 @@ def test___version__(): def test_dev_utils(): - import dpctl._dev as dd + import tempfile - with dd.verbose(): + import dpctl._diagnostics as dd + + ctx_mngr = dd.syclinterface_diagnostics + + with ctx_mngr(): dpctl.SyclDevice().parent_device - with dd.verbose(verbosity="error"): + with ctx_mngr(verbosity="error"): dpctl.SyclDevice().parent_device with pytest.raises(ValueError): - with dd.verbose(verbosity="blah"): + with ctx_mngr(verbosity="blah"): + dpctl.SyclDevice().parent_device + with tempfile.TemporaryDirectory() as temp_dir: + with ctx_mngr(log_dir=temp_dir): dpctl.SyclDevice().parent_device - with dd.verbose(log_dir="/tmp"): - dpctl.SyclDevice().parent_device with pytest.raises(ValueError): - with dd.verbose(log_dir="/not_a_dir"): + with ctx_mngr(log_dir="/not_a_dir"): dpctl.SyclDevice().parent_device diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp index 3a5752e55b..c880bb52ff 100644 --- a/libsyclinterface/helper/source/dpctl_error_handlers.cpp +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -65,21 +65,30 @@ int requested_verbosity_level(void) return requested_level; } -void output_message(std::string ss_str, [[maybe_unused]] error_level error_type) +void output_message(std::string ss_str, error_level error_type) { #ifdef ENABLE_GLOG switch (error_type) { case error_level::error: - LOG(ERROR) << ss_str; + LOG(ERROR) << "[ERR] " << ss_str; break; case error_level::warning: - LOG(WARNING) << ss_str; + LOG(WARNING) << "[WARN] " << ss_str; break; default: - LOG(FATAL) << ss_str; + LOG(FATAL) << "[FATAL] " << ss_str; } #else - std::cerr << ss_str; + switch (error_type) { + case error_level::error: + std::cerr << "[ERR] " << ss_str; + break; + case error_level::warning: + std::cerr << "[WARN] " << ss_str; + break; + default: + std::cerr << "[FATAL] " << ss_str; + } #endif } diff --git a/libsyclinterface/source/dpctl_service.cpp b/libsyclinterface/source/dpctl_service.cpp index 944786e1a4..7e96b6119d 100644 --- a/libsyclinterface/source/dpctl_service.cpp +++ b/libsyclinterface/source/dpctl_service.cpp @@ -46,21 +46,22 @@ __dpctl_give const char *DPCTLService_GetDPCPPVersion(void) void DPCTLService_InitLogger(const char *app_name, const char *log_dir) { google::InitGoogleLogging(app_name); - google::EnableLogCleaner(0); google::InstallFailureSignalHandler(); - FLAGS_colorlogtostderr = true; - FLAGS_stderrthreshold = google::FATAL; - namespace fs = std::filesystem; - const fs::path path(log_dir); - std::error_code ec; + if (log_dir) { + namespace fs = std::filesystem; + const fs::path path(log_dir); + std::error_code ec; - if (fs::is_directory(path, ec)) { - FLAGS_log_dir = log_dir; + if (fs::is_directory(path, ec)) { + google::EnableLogCleaner(0); + FLAGS_log_dir = log_dir; + } } else { - std::cerr << "Directory not found. Log files will be created in the " - "default logging directory.\n"; + FLAGS_colorlogtostderr = true; + FLAGS_stderrthreshold = google::FATAL; + FLAGS_logtostderr = 1; } } From 84e7dc57671f9e47df8d7f82f8bc014bdc1472ba Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 26 Jan 2022 15:00:24 +0300 Subject: [PATCH 190/229] Update CONTRIBUTING.md --- CONTRIBUTING.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1bf79975e..f8d961169b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -161,13 +161,12 @@ these steps: 3. Build dpctl with code coverage support. ```bash - python setup.py develop --coverage=True - pytest -q -ra --disable-warnings --cov dpctl --cov-report term-missing --pyargs dpctl -vv + python scripts/gen_coverage.py --oneapi coverage html ``` Note that code coverage builds the C sources with debug symbols. For this - reason, the coverage flag is only available with the `develop` mode of + reason, the coverage script builds the package in `develop` mode of `setup.py`. The coverage results for the C and Python sources will be printed to the @@ -191,3 +190,52 @@ these steps: > ``` > The error is related to the `tcl` package. You should uninstall the `tcl` > package to resolve the error. + +## Error Reporting and Logging + +The SyclInterface library responds to `DPCTL_VERBOSITY` environment variable that controls the severity level of errors printed to console. +One can specify one of the following severity levels (in increasing order of severity): `warning` and `error`. + +```bash +export DPCTL_VERBOSITY=warning +``` + +Messages of a given severity are shown not only in the console for that severity, but also for the higher severity. For example, the severity level `warning` will output severity errors for `error` and `warning` to the console. + +### Optional use of the Google logging library (glog) + +Dpctl's error handler for libsyclinterface can be optionally configured to use [glog](https://github.com/google/glog). To use glog, follow the following steps: + +1. Install glog package of the latest version (0.5.0) + +```bash +conda install glog +``` +2. Build dpctl with glog support + +```bash +python scripts/build_locally.py --oneapi --glog +``` + +3. Use `dpctl._diagnostics.syclinterface_diagnostics(verbosity="warning", log_dir=None)` context manager to switch library diagnostics on for a block of Python code. +Use `DPCTLService_InitLogger` and `DPCTLService_ShutdownLogger` library C functions during library development to initialize the Google's logging library and de-initialize accordingly + +```python +from dpctl._diagnostics import syclinterface_diagnostics +import dpctl + +with syclinterface_diagnostics(): + code +``` + +```c +DPCTLService_InitLogger(const char *app_name, const char *log_dir); +DPCTLService_ShutdownLogger(); +``` + + - `*app_name` - name of the executable file (prefix for logs of various levels). + - `*log_dir` - directory path for writing log files. Specifying `NULL` results in logging to ``std::cerr``. + +> **_NOTE:_** +> +> If `InitGoogleLogging` is not called before first use of glog, the library will self-initialize to `logtostderr` mode and log files will not be generated. From c4c5e4c6cf2109793074c683d7e3f6ef280dd2d7 Mon Sep 17 00:00:00 2001 From: Perevezentsev Vladislav Date: Sun, 30 Jan 2022 19:44:27 +0300 Subject: [PATCH 191/229] Add Windows support to build_locally.py --- scripts/build_locally.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 049079cc37..b2168c6fe8 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -11,19 +11,15 @@ def run( compiler_root=None, cmake_executable=None, ): - IS_LIN = False + build_system = None if "linux" in sys.platform: - IS_LIN = True + build_system = "Unix Makefiles" elif sys.platform in ["win32", "cygwin"]: - pass + build_system = "Ninja" else: assert False, sys.platform + " not supported" - if not IS_LIN: - raise RuntimeError( - "This scripts only supports coverage collection on Linux" - ) setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cmake_args = [ sys.executable, @@ -37,7 +33,7 @@ def run( cmake_args += [ "--", "-G", - "Unix Makefiles", + build_system, "-DCMAKE_BUILD_TYPE=Debug", "-DCMAKE_C_COMPILER:PATH=" + c_compiler, "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, @@ -86,7 +82,7 @@ def run( if args.oneapi: args.c_compiler = "icx" - args.cxx_compiler = "icpx" + args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" args.compiler_root = None else: args_to_validate = [ From a5e127e0648492e09b25e32839daaa6de90568d5 Mon Sep 17 00:00:00 2001 From: Perevezentsev Vladislav Date: Mon, 31 Jan 2022 16:54:37 +0300 Subject: [PATCH 192/229] Add --debug option to command line argument --- scripts/build_locally.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index b2168c6fe8..1a3927433d 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -5,6 +5,7 @@ def run( use_oneapi=True, + build_type="Relese", c_compiler=None, cxx_compiler=None, level_zero=True, @@ -34,7 +35,7 @@ def run( "--", "-G", build_system, - "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_BUILD_TYPE=" + build_type, "-DCMAKE_C_COMPILER:PATH=" + c_compiler, "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), @@ -66,6 +67,13 @@ def run( dest="oneapi", action="store_true", ) + driver.add_argument( + "--debug", + default="Release", + const="Debug", + action="store_const", + help="Set the compilation mode to debugging", + ) driver.add_argument( "--compiler-root", type=str, help="Path to compiler home directory" ) @@ -103,6 +111,7 @@ def run( run( use_oneapi=args.oneapi, + build_type=args.debug, c_compiler=args.c_compiler, cxx_compiler=args.cxx_compiler, level_zero=args.level_zero, From 4e38ae1af9574e20904a4d9c1b4d35f221a86fc1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 31 Jan 2022 15:08:57 -0600 Subject: [PATCH 193/229] Update scripts/build_locally.py Fixed typo --- scripts/build_locally.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 1a3927433d..32cd46792d 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -5,7 +5,7 @@ def run( use_oneapi=True, - build_type="Relese", + build_type="Release", c_compiler=None, cxx_compiler=None, level_zero=True, From 3048f3e577ed6ae64ee82d247a5e9a72d6ecb3ae Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 05:56:38 -0600 Subject: [PATCH 194/229] Fixes #729 The generated reshaped_strides routine is meant for non-empty arrays, and so empty ones must be handled differently. Tests added as well. --- dpctl/tensor/_reshape.py | 5 ++++- dpctl/tests/test_usm_ndarray_ctor.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dpctl/tensor/_reshape.py b/dpctl/tensor/_reshape.py index 8e04f1b20c..ffa60c3652 100644 --- a/dpctl/tensor/_reshape.py +++ b/dpctl/tensor/_reshape.py @@ -104,7 +104,10 @@ def reshape(X, newshape, order="C"): newshape = [v if d == -1 else d for d in newshape] if X.size != np.prod(newshape): raise ValueError("Can not reshape into {}".format(newshape)) - newsts = reshaped_strides(X.shape, X.strides, newshape, order=order) + if X.size: + newsts = reshaped_strides(X.shape, X.strides, newshape, order=order) + else: + newsts = (1,) * len(newshape) if newsts is None: # must perform a copy flat_res = dpt.usm_ndarray( diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 410c3128e8..ca2dac41f8 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -816,6 +816,32 @@ def test_reshape(): Y = dpt.reshape(X, X.shape) assert Y.flags == X.flags + A = dpt.usm_ndarray((0,), "i4") + A1 = dpt.reshape(A, (0,)) + assert A1.shape == (0,) + A2 = dpt.reshape( + A, + ( + 2, + 0, + ), + ) + assert A2.shape == ( + 2, + 0, + ) + A3 = dpt.reshape(A, (0, 2)) + assert A3.shape == ( + 0, + 2, + ) + A4 = dpt.reshape(A, (1, 0, 2)) + assert A4.shape == ( + 1, + 0, + 2, + ) + def test_transpose(): n, m = 2, 3 From 479ed600e9c5cbcbd8fc74132d70d3025767382f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 06:26:50 -0600 Subject: [PATCH 195/229] reshaped_strides is also called from shape setter Special case setting shape for zero-element arrays --- dpctl/tensor/_usmarray.pyx | 17 +++++++++++------ dpctl/tests/test_usm_ndarray_ctor.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 959164dd3c..3d6d16c5b5 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -428,6 +428,7 @@ cdef class usm_ndarray: cdef int contig_flag = 0 cdef Py_ssize_t *shape_ptr = NULL cdef Py_ssize_t *strides_ptr = NULL + cdef Py_ssize_t size = -1 import operator from ._reshape import reshaped_strides @@ -439,15 +440,19 @@ cdef class usm_ndarray: raise TypeError( "Target shape must be a finite iterable of integers" ) - if not np.prod(new_shape) == shape_to_elem_count(self.nd_, self.shape_): + size = shape_to_elem_count(self.nd_, self.shape_) + if not np.prod(new_shape) == size: raise TypeError( f"Can not reshape array of size {self.size} into {new_shape}" ) - new_strides = reshaped_strides( - self.shape, - self.strides, - new_shape - ) + if size > 0: + new_strides = reshaped_strides( + self.shape, + self.strides, + new_shape + ) + else: + new_strides = (1,) * len(new_shape) if new_strides is None: raise AttributeError( "Incompatible shape for in-place modification. " diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index ca2dac41f8..e8ee79a391 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -713,6 +713,21 @@ def relaxed_strides_equal(st1, st2, sh): X = dpt.usm_ndarray((4, 4), dtype="d")[::2, ::2] with pytest.raises(AttributeError): X.shape = (4,) + X = dpt.usm_ndarray((0,), dtype="i4") + X.shape = (0,) + X.shape = ( + 2, + 0, + ) + X.shape = ( + 0, + 2, + ) + X.shape = ( + 1, + 0, + 1, + ) def test_len(): From 624564e01b5e4e31ec7c2b5b4a443c70e1afba1a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 06:15:55 -0600 Subject: [PATCH 196/229] Fixed #728 Special-cased copying of 0-element arrays. --- dpctl/tensor/_copy_utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index c49862d70f..87c7179dcc 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -66,15 +66,20 @@ def contract_iter2(shape, strides1, strides2): def _has_memory_overlap(x1, x2): - m1 = dpm.as_usm_memory(x1) - m2 = dpm.as_usm_memory(x2) - if m1.sycl_device == m2.sycl_device: - p1_beg = m1._pointer - p1_end = p1_beg + m1.nbytes - p2_beg = m2._pointer - p2_end = p2_beg + m2.nbytes - return p1_beg > p2_end or p2_beg < p1_end + if x1.size and x2.size: + m1 = dpm.as_usm_memory(x1) + m2 = dpm.as_usm_memory(x2) + # can only overlap if bound to the same context + if m1.sycl_context == m2.sycl_context: + p1_beg = m1._pointer + p1_end = p1_beg + m1.nbytes + p2_beg = m2._pointer + p2_end = p2_beg + m2.nbytes + return p1_beg > p2_end or p2_beg < p1_end + else: + return False else: + # zero element array do not overlap anything return False @@ -193,6 +198,9 @@ def copy_same_dtype(dst, src): if dst.dtype != src.dtype: raise ValueError + if dst.size == 0: + return + # check that memory regions do not overlap if _has_memory_overlap(dst, src): tmp = _copy_to_numpy(src) From 3ce1b68b66741f9d3c330f35dfe09eca7fdd536a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 07:11:24 -0600 Subject: [PATCH 197/229] Tests added for problem from #728 ```python In [1]: import dpctl.tensor as dpt In [2]: X = dpt.asarray([]) In [3]: dpt.astype(X, float) Out[3]: In [4]: (dpt.asnumpy(_), _.dtype, _.shape, _.strides) Out[4]: (array([], dtype=float64), dtype('float64'), (0,), (1,)) ``` --- dpctl/tests/test_usm_ndarray_ctor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index e8ee79a391..c9e22ece41 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -600,6 +600,8 @@ def test_setitem_same_dtype(dtype, src_usm_type, dst_usm_type): R2 = np.broadcast_to(Xnp[0], R1.shape) assert R1.shape == R2.shape assert np.allclose(R1, R2) + Zusm_empty = Zusm_1d[0:0] + Zusm_empty[Ellipsis] = Zusm_3d[0, 0, 0:0] @pytest.mark.parametrize( From acad115d9d9d2489d822cf3a88d2563765af6cb3 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 14:49:47 -0600 Subject: [PATCH 198/229] Do not use relative path in include statements which presumes rigid folder structure Compiler's include directory options are used to direct compiler to location of header files --- libsyclinterface/helper/include/dpctl_string_utils.hpp | 2 +- libsyclinterface/source/dpctl_sycl_context_interface.cpp | 2 +- libsyclinterface/source/dpctl_sycl_device_interface.cpp | 6 +++--- libsyclinterface/source/dpctl_sycl_device_manager.cpp | 6 +++--- .../source/dpctl_sycl_device_selector_interface.cpp | 2 +- libsyclinterface/source/dpctl_sycl_event_interface.cpp | 4 ++-- libsyclinterface/source/dpctl_sycl_kernel_interface.cpp | 4 ++-- libsyclinterface/source/dpctl_sycl_platform_interface.cpp | 6 +++--- libsyclinterface/source/dpctl_sycl_platform_manager.cpp | 4 ++-- libsyclinterface/source/dpctl_sycl_program_interface.cpp | 2 +- libsyclinterface/source/dpctl_sycl_queue_interface.cpp | 2 +- libsyclinterface/source/dpctl_sycl_queue_manager.cpp | 2 +- libsyclinterface/source/dpctl_sycl_usm_interface.cpp | 2 +- libsyclinterface/source/dpctl_vector_templ.cpp | 4 ++-- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libsyclinterface/helper/include/dpctl_string_utils.hpp b/libsyclinterface/helper/include/dpctl_string_utils.hpp index 735f972128..de13003813 100644 --- a/libsyclinterface/helper/include/dpctl_string_utils.hpp +++ b/libsyclinterface/helper/include/dpctl_string_utils.hpp @@ -21,7 +21,7 @@ /// \file /// Helper function to convert a C++ string to a C string. //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_error_handlers.h" +#include "dpctl_error_handlers.h" #include #include #include diff --git a/libsyclinterface/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp index 62d488d935..2398264785 100644 --- a/libsyclinterface/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -25,8 +25,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_context_interface.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include #include diff --git a/libsyclinterface/source/dpctl_sycl_device_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_interface.cpp index 804998cf19..47109fd21c 100644 --- a/libsyclinterface/source/dpctl_sycl_device_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_interface.cpp @@ -25,11 +25,11 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_interface.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include "dpctl_sycl_device_manager.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include #include diff --git a/libsyclinterface/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp index bb140dbc0c..7a81b828b2 100644 --- a/libsyclinterface/source/dpctl_sycl_device_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_manager.cpp @@ -24,11 +24,11 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_manager.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include "dpctl_sycl_enum_types.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include #include diff --git a/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp index 8793775806..fb87980aaa 100644 --- a/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp @@ -24,8 +24,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_selector_interface.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include /* SYCL headers */ using namespace cl::sycl; diff --git a/libsyclinterface/source/dpctl_sycl_event_interface.cpp b/libsyclinterface/source/dpctl_sycl_event_interface.cpp index 2ba4b28011..406a873258 100644 --- a/libsyclinterface/source/dpctl_sycl_event_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_event_interface.cpp @@ -25,9 +25,9 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_event_interface.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include diff --git a/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp index a1a7e38e39..48d8ff919e 100644 --- a/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp @@ -25,9 +25,9 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_kernel_interface.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_string_utils.hpp" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include /* Sycl headers */ using namespace cl::sycl; diff --git a/libsyclinterface/source/dpctl_sycl_platform_interface.cpp b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp index 1ae656b08f..36c9bd16b7 100644 --- a/libsyclinterface/source/dpctl_sycl_platform_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp @@ -25,10 +25,10 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_interface.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" +#include "dpctl_utils_helper.h" #include #include #include diff --git a/libsyclinterface/source/dpctl_sycl_platform_manager.cpp b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp index 679b0c6350..90dcf0ee71 100644 --- a/libsyclinterface/source/dpctl_sycl_platform_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp @@ -25,10 +25,10 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_manager.h" -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_platform_interface.h" +#include "dpctl_utils_helper.h" #include #include #include diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index b01e82e8e1..3bc98bfcb7 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -30,9 +30,9 @@ #endif #include "dpctl_sycl_program_interface.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Config/dpctl_config.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include /* OpenCL headers */ #include /* Sycl headers */ #include diff --git a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp index 152bec562d..c0bf550a63 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp @@ -25,8 +25,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_interface.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_context_interface.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_manager.h" diff --git a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp index 5cd924bdd4..15379e07b3 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp @@ -24,8 +24,8 @@ /// //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_manager.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_device_manager.h" #include /* SYCL headers */ #include diff --git a/libsyclinterface/source/dpctl_sycl_usm_interface.cpp b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp index 2347898c36..e6e7b2fcf0 100644 --- a/libsyclinterface/source/dpctl_sycl_usm_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp @@ -25,8 +25,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_usm_interface.h" -#include "../helper/include/dpctl_error_handlers.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_device_interface.h" #include /* SYCL headers */ diff --git a/libsyclinterface/source/dpctl_vector_templ.cpp b/libsyclinterface/source/dpctl_vector_templ.cpp index a2f9e8def9..077b6ad45d 100644 --- a/libsyclinterface/source/dpctl_vector_templ.cpp +++ b/libsyclinterface/source/dpctl_vector_templ.cpp @@ -23,9 +23,9 @@ /// the wrapper functions for vector operations. /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_error_handlers.h" -#include "../helper/include/dpctl_vector_macros.h" #include "Support/MemOwnershipAttrs.h" +#include "dpctl_error_handlers.h" +#include "dpctl_vector_macros.h" #include #include From 550c6f090a84540714b1f51df651775742bb13c0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 15:09:19 -0600 Subject: [PATCH 199/229] backend::level_zero -> backend::ext_oneapi_level_zero --- libsyclinterface/helper/source/dpctl_utils_helper.cpp | 4 ++-- libsyclinterface/source/dpctl_sycl_context_interface.cpp | 2 +- libsyclinterface/source/dpctl_sycl_program_interface.cpp | 9 +++++---- libsyclinterface/tests/test_helper.cpp | 8 ++++---- libsyclinterface/tests/test_sycl_queue_interface.cpp | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp index bf3f590bc6..599d2be29e 100644 --- a/libsyclinterface/helper/source/dpctl_utils_helper.cpp +++ b/libsyclinterface/helper/source/dpctl_utils_helper.cpp @@ -93,7 +93,7 @@ backend DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType BeTy) case DPCTLSyclBackendType::DPCTL_HOST: return backend::host; case DPCTLSyclBackendType::DPCTL_LEVEL_ZERO: - return backend::level_zero; + return backend::ext_oneapi_level_zero; case DPCTLSyclBackendType::DPCTL_OPENCL: return backend::opencl; case DPCTLSyclBackendType::DPCTL_ALL_BACKENDS: @@ -110,7 +110,7 @@ DPCTLSyclBackendType DPCTL_SyclBackendToDPCTLBackendType(backend B) return DPCTLSyclBackendType::DPCTL_CUDA; case backend::host: return DPCTLSyclBackendType::DPCTL_HOST; - case backend::level_zero: + case backend::ext_oneapi_level_zero: return DPCTLSyclBackendType::DPCTL_LEVEL_ZERO; case backend::opencl: return DPCTLSyclBackendType::DPCTL_OPENCL; diff --git a/libsyclinterface/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp index 2398264785..157b122236 100644 --- a/libsyclinterface/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -194,7 +194,7 @@ DPCTLContext_GetBackend(__dpctl_keep const DPCTLSyclContextRef CtxRef) return DPCTL_HOST; case backend::opencl: return DPCTL_OPENCL; - case backend::level_zero: + case backend::ext_oneapi_level_zero: return DPCTL_LEVEL_ZERO; case backend::cuda: return DPCTL_CUDA; diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index 3bc98bfcb7..e9ecb42cc6 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -146,7 +146,7 @@ createLevelZeroInterOpProgram(const context &SyclCtx, size_t length, const char *CompileOpts) { - auto ZeCtx = get_native(SyclCtx); + auto ZeCtx = get_native(SyclCtx); auto SyclDevices = SyclCtx.get_devices(); if (SyclDevices.size() > 1) { error_handler("Level zero program can be created for only one device.", @@ -168,7 +168,7 @@ createLevelZeroInterOpProgram(const context &SyclCtx, ZeModuleDesc.pBuildFlags = CompileOpts; ZeModuleDesc.pConstants = &ZeSpecConstants; - auto ZeDevice = get_native(SyclDevices[0]); + auto ZeDevice = get_native(SyclDevices[0]); ze_module_handle_t ZeModule; auto stZeModuleCreateF = getZeModuleCreateFn(); @@ -189,8 +189,9 @@ createLevelZeroInterOpProgram(const context &SyclCtx, // Create the Sycl program from the ZeModule try { - auto ZeProgram = new program(sycl::level_zero::make_program( - SyclCtx, reinterpret_cast(ZeModule))); + auto ZeProgram = + new program(sycl::ext::oneapi::level_zero::make_program( + SyclCtx, reinterpret_cast(ZeModule))); return wrap(ZeProgram); } catch (std::exception const &e) { error_handler(e, __FILE__, __func__, __LINE__); diff --git a/libsyclinterface/tests/test_helper.cpp b/libsyclinterface/tests/test_helper.cpp index ae4f60b75c..ea7d638aab 100644 --- a/libsyclinterface/tests/test_helper.cpp +++ b/libsyclinterface/tests/test_helper.cpp @@ -86,7 +86,7 @@ TEST_F(TestHelperFns, ChkStrToDeviceType) TEST_F(TestHelperFns, ChkDPCTLBackendTypeToSyclBackend) { - sycl::backend res = sycl::backend::level_zero; + sycl::backend res = sycl::backend::ext_oneapi_level_zero; EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( DPCTLSyclBackendType::DPCTL_CUDA)); @@ -102,7 +102,7 @@ TEST_F(TestHelperFns, ChkDPCTLBackendTypeToSyclBackend) EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( DPCTLSyclBackendType::DPCTL_LEVEL_ZERO)); - ASSERT_TRUE(res == sycl::backend::level_zero); + ASSERT_TRUE(res == sycl::backend::ext_oneapi_level_zero); EXPECT_THROW(DPCTL_DPCTLBackendTypeToSyclBackend( DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND), @@ -113,8 +113,8 @@ TEST_F(TestHelperFns, ChkSyclBackendToDPCTLBackendType) { DPCTLSyclBackendType DTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; - EXPECT_NO_FATAL_FAILURE( - DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::level_zero)); + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclBackendToDPCTLBackendType( + sycl::backend::ext_oneapi_level_zero)); ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_LEVEL_ZERO); EXPECT_NO_FATAL_FAILURE( diff --git a/libsyclinterface/tests/test_sycl_queue_interface.cpp b/libsyclinterface/tests/test_sycl_queue_interface.cpp index a75262eb20..fa8c3cd4f1 100644 --- a/libsyclinterface/tests/test_sycl_queue_interface.cpp +++ b/libsyclinterface/tests/test_sycl_queue_interface.cpp @@ -358,7 +358,7 @@ TEST_P(TestDPCTLQueueMemberFunctions, CheckGetBackend) EXPECT_TRUE(Backend == backend::host); break; case DPCTL_LEVEL_ZERO: - EXPECT_TRUE(Backend == backend::level_zero); + EXPECT_TRUE(Backend == backend::ext_oneapi_level_zero); break; case DPCTL_OPENCL: EXPECT_TRUE(Backend == backend::opencl); From 56ba791c96546fa308580cae8914c04201b09c80 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 17:57:19 -0600 Subject: [PATCH 200/229] addressed sycl deprecation warnings. Changes DPCTL aspect enum to align with SYCL aspect renaming: usm_system_allocators->usm_system_allocations --- .../helper/source/dpctl_error_handlers.cpp | 2 +- .../helper/source/dpctl_utils_helper.cpp | 16 ++++++++-------- libsyclinterface/include/dpctl_sycl_enum_types.h | 2 +- .../source/dpctl_sycl_program_interface.cpp | 4 ++-- .../source/dpctl_sycl_queue_interface.cpp | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp index c880bb52ff..f94ed129f3 100644 --- a/libsyclinterface/helper/source/dpctl_error_handlers.cpp +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -39,7 +39,7 @@ void DPCTL_AsyncErrorHandler::operator()( std::rethrow_exception(e); } catch (sycl::exception const &e) { error_handler(e, __FILE__, __func__, __LINE__); - auto err_code = e.get_cl_code(); + auto err_code = e.code(); handler_(static_cast(err_code)); } } diff --git a/libsyclinterface/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp index 599d2be29e..c0013ddb71 100644 --- a/libsyclinterface/helper/source/dpctl_utils_helper.cpp +++ b/libsyclinterface/helper/source/dpctl_utils_helper.cpp @@ -221,8 +221,8 @@ std::string DPCTL_AspectToStr(aspect aspectTy) case aspect::usm_restricted_shared_allocations: ss << "usm_restricted_shared_allocations"; break; - case aspect::usm_system_allocator: - ss << "usm_system_allocator"; + case aspect::usm_system_allocations: + ss << "usm_system_allocations"; break; default: throw std::runtime_error("Unsupported aspect type"); @@ -287,8 +287,8 @@ aspect DPCTL_StrToAspectType(const std::string &aspectTyStr) else if (aspectTyStr == "usm_restricted_shared_allocations") { aspectTy = aspect::usm_restricted_shared_allocations; } - else if (aspectTyStr == "usm_system_allocator") { - aspectTy = aspect::usm_system_allocator; + else if (aspectTyStr == "usm_system_allocations") { + aspectTy = aspect::usm_system_allocations; } else { // \todo handle the error @@ -334,8 +334,8 @@ aspect DPCTL_DPCTLAspectTypeToSyclAspect(DPCTLSyclAspectType AspectTy) return aspect::usm_shared_allocations; case DPCTLSyclAspectType::usm_restricted_shared_allocations: return aspect::usm_restricted_shared_allocations; - case DPCTLSyclAspectType::usm_system_allocator: - return aspect::usm_system_allocator; + case DPCTLSyclAspectType::usm_system_allocations: + return aspect::usm_system_allocations; default: throw std::runtime_error("Unsupported aspect type"); } @@ -378,8 +378,8 @@ DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(aspect Aspect) return DPCTLSyclAspectType::usm_shared_allocations; case aspect::usm_restricted_shared_allocations: return DPCTLSyclAspectType::usm_restricted_shared_allocations; - case aspect::usm_system_allocator: - return DPCTLSyclAspectType::usm_system_allocator; + case aspect::usm_system_allocations: + return DPCTLSyclAspectType::usm_system_allocations; default: throw std::runtime_error("Unsupported aspect type"); } diff --git a/libsyclinterface/include/dpctl_sycl_enum_types.h b/libsyclinterface/include/dpctl_sycl_enum_types.h index 86f1855129..afb97905ef 100644 --- a/libsyclinterface/include/dpctl_sycl_enum_types.h +++ b/libsyclinterface/include/dpctl_sycl_enum_types.h @@ -119,7 +119,7 @@ typedef enum usm_host_allocations, usm_shared_allocations, usm_restricted_shared_allocations, - usm_system_allocator + usm_system_allocations } DPCTLSyclAspectType; /*! diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index e9ecb42cc6..d081e4fc4d 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -223,7 +223,7 @@ DPCTLProgram_CreateFromSpirv(__dpctl_keep const DPCTLSyclContextRef CtxRef, case backend::opencl: Pref = createOpenCLInterOpProgram(*SyclCtx, IL, length, CompileOpts); break; - case backend::level_zero: + case backend::ext_oneapi_level_zero: #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION Pref = createLevelZeroInterOpProgram(*SyclCtx, IL, length, CompileOpts); #endif @@ -274,7 +274,7 @@ DPCTLProgram_CreateFromOCLSource(__dpctl_keep const DPCTLSyclContextRef Ctx, return nullptr; } break; - case backend::level_zero: + case backend::ext_oneapi_level_zero: error_handler("CreateFromSource is not supported in Level Zero.", __FILE__, __func__, __LINE__); delete SyclProgram; diff --git a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp index c0bf550a63..61630809d6 100644 --- a/libsyclinterface/source/dpctl_sycl_queue_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp @@ -530,7 +530,7 @@ DPCTLSyclEventRef DPCTLQueue_MemAdvise(__dpctl_keep DPCTLSyclQueueRef QRef, if (Q) { sycl::event ev; try { - ev = Q->mem_advise(Ptr, Count, static_cast(Advice)); + ev = Q->mem_advise(Ptr, Count, Advice); } catch (std::exception const &e) { error_handler(e, __FILE__, __func__, __LINE__); return nullptr; @@ -592,7 +592,7 @@ __dpctl_give DPCTLSyclEventRef DPCTLQueue_SubmitBarrierForEvents( for (auto i = 0ul; i < NDepEvents; ++i) cgh.depends_on(*unwrap(DepEvents[i])); - cgh.barrier(); + cgh.ext_oneapi_barrier(); }); } catch (std::exception const &e) { error_handler(e, __FILE__, __func__, __LINE__); From e612cb0148ea443ccd6c57a1ae746f5519715e02 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 1 Feb 2022 18:36:59 -0600 Subject: [PATCH 201/229] Updated deprecated declarations with their replacements The only outstanding deprecation is use of `sycl::program`, but addressing that would take more work to fix. Since `sycl::aspect::usm_system_allocator` was deprecated in favor of `sycl::aspect::usm_system_allocations` in line with other aspects, the DPCTL enum was renamed accordingly. Tests were changed. This change is user visible. --- dpctl/_backend.pxd | 2 +- dpctl/_sycl_device.pyx | 4 ++-- dpctl/tests/test_sycl_device.py | 10 +++++----- dpctl/tests/test_sycl_queue.py | 8 ++++---- .../helper/source/dpctl_error_handlers.cpp | 2 +- libsyclinterface/tests/test_sycl_device_aspects.cpp | 6 +++--- libsyclinterface/tests/test_sycl_device_subdevices.cpp | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index b4788ab493..8a666b045b 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -93,7 +93,7 @@ cdef extern from "syclinterface/dpctl_sycl_enum_types.h": _usm_host_allocations 'usm_host_allocations', _usm_shared_allocations 'usm_shared_allocations', _usm_restricted_shared_allocations 'usm_restricted_shared_allocations', - _usm_system_allocator 'usm_system_allocator' + _usm_system_allocations 'usm_system_allocations' ctypedef enum _partition_affinity_domain_type 'DPCTLPartitionAffinityDomainType': _not_applicable 'not_applicable', diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 2fc00763b0..ee30002cce 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -441,8 +441,8 @@ cdef class SyclDevice(_SyclDevice): return DPCTLDevice_HasAspect(self._device_ref, AT) @property - def has_aspect_usm_system_allocator(self): - cdef _aspect_type AT = _aspect_type._usm_system_allocator + def has_aspect_usm_system_allocations(self): + cdef _aspect_type AT = _aspect_type._usm_system_allocations return DPCTLDevice_HasAspect(self._device_ref, AT) @property diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 409c5c5c13..6277504ef2 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -219,11 +219,11 @@ def check_has_aspect_usm_restricted_shared_allocations(device): pytest.fail("has_aspect_usm_restricted_shared_allocations call failed") -def check_has_aspect_usm_system_allocator(device): +def check_has_aspect_usm_system_allocations(device): try: - device.has_aspect_usm_system_allocator + device.has_aspect_usm_system_allocations except Exception: - pytest.fail("has_aspect_usm_system_allocator call failed") + pytest.fail("has_aspect_usm_system_allocations call failed") def check_is_accelerator(device): @@ -520,7 +520,7 @@ def check_repr(device): check_has_aspect_usm_host_allocations, check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, - check_has_aspect_usm_system_allocator, + check_has_aspect_usm_system_allocations, check_get_max_read_image_args, check_get_max_write_image_args, check_get_image_2d_max_width, @@ -679,7 +679,7 @@ def test_hashing_of_device(): "usm_device_allocations", "usm_host_allocations", "usm_shared_allocations", - "usm_system_allocator", + "usm_system_allocations", ] # SYCL 2020 spec aspects not presently diff --git a/dpctl/tests/test_sycl_queue.py b/dpctl/tests/test_sycl_queue.py index 3d3a165d4b..6ee176a726 100644 --- a/dpctl/tests/test_sycl_queue.py +++ b/dpctl/tests/test_sycl_queue.py @@ -211,11 +211,11 @@ def check_has_aspect_usm_restricted_shared_allocations(device): pytest.fail("has_aspect_usm_restricted_shared_allocations call failed") -def check_has_aspect_usm_system_allocator(device): +def check_has_aspect_usm_system_allocations(device): try: - device.has_aspect_usm_system_allocator + device.has_aspect_usm_system_allocations except Exception: - pytest.fail("has_aspect_usm_system_allocator call failed") + pytest.fail("has_aspect_usm_system_allocations call failed") def check_is_accelerator(device): @@ -273,7 +273,7 @@ def check_is_host(device): check_has_aspect_usm_host_allocations, check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, - check_has_aspect_usm_system_allocator, + check_has_aspect_usm_system_allocations, ] diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp index f94ed129f3..e6b8adeabf 100644 --- a/libsyclinterface/helper/source/dpctl_error_handlers.cpp +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -39,7 +39,7 @@ void DPCTL_AsyncErrorHandler::operator()( std::rethrow_exception(e); } catch (sycl::exception const &e) { error_handler(e, __FILE__, __func__, __LINE__); - auto err_code = e.code(); + auto err_code = e.code().value(); handler_(static_cast(err_code)); } } diff --git a/libsyclinterface/tests/test_sycl_device_aspects.cpp b/libsyclinterface/tests/test_sycl_device_aspects.cpp index 90b29a8bac..60a7acad77 100644 --- a/libsyclinterface/tests/test_sycl_device_aspects.cpp +++ b/libsyclinterface/tests/test_sycl_device_aspects.cpp @@ -99,8 +99,8 @@ auto build_params() cl::sycl::aspect::usm_shared_allocations), std::make_pair("usm_restricted_shared_allocations", cl::sycl::aspect::usm_restricted_shared_allocations), - std::make_pair("usm_system_allocator", - cl::sycl::aspect::usm_system_allocator)); + std::make_pair("usm_system_allocations", + cl::sycl::aspect::usm_system_allocations)); auto pairs = build_param_pairshas(syclAspect); - } catch (cl::sycl::runtime_error const &re) { + } catch (sycl::exception const &e) { } } diff --git a/libsyclinterface/tests/test_sycl_device_subdevices.cpp b/libsyclinterface/tests/test_sycl_device_subdevices.cpp index 41cef23784..ebf76fe076 100644 --- a/libsyclinterface/tests/test_sycl_device_subdevices.cpp +++ b/libsyclinterface/tests/test_sycl_device_subdevices.cpp @@ -149,7 +149,7 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityNotApplicable) EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } - } catch (runtime_error const &re) { + } catch (exception const &e) { } } } From ec6a1ad75dd83d58573ac83a8c0a0659ec29a44b Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 2 Feb 2022 00:18:02 -0600 Subject: [PATCH 202/229] Add missing license headers. --- scripts/build_locally.py | 16 ++++++++++++++++ scripts/gen_coverage.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 049079cc37..938b8c42d9 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -1,3 +1,19 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import subprocess import sys diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 68ad75ea52..43cffd1cd9 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -1,3 +1,19 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import subprocess import sys From 89b955f20ea0a0d79d640f84fa1ff6ad5f506b5b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 2 Feb 2022 09:17:42 -0600 Subject: [PATCH 203/229] Replaced one stray #include `"../helper/include/header.h"` with `#include "header.h"` Replaced `#include ` with `#include ` and applied clang-format guards per comment in the code. --- libsyclinterface/source/dpctl_sycl_program_interface.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libsyclinterface/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp index d081e4fc4d..f5369f94ae 100644 --- a/libsyclinterface/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -39,11 +39,13 @@ #include #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION -#include "../helper/include/dpctl_dynamic_lib_helper.h" -#include /* Level Zero headers */ +#include "dpctl_dynamic_lib_helper.h" // Note: include ze_api.h before level_zero.hpp. Make sure clang-format does // not reorder the includes. -#include +// clang-format off +#include "ze_api.h" /* Level Zero headers */ +#include "sycl/ext/oneapi/backend/level_zero.hpp" +// clang-format on #endif using namespace cl::sycl; From dc14fd26aa900b4c23c91db1aeffdbf985ed7310 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 2 Feb 2022 19:32:56 -0600 Subject: [PATCH 204/229] Fixed CMakeLists in docs to work with top-level cmake. Add generator script. --- docs/CMakeLists.txt | 13 +--- scripts/gen_docs.py | 171 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 scripts/gen_docs.py diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 82e8e1a0e1..ff3fbcb7cd 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -1,6 +1,3 @@ -cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -project("Data-parallel Control (dpctl)") - # Option to generate rst for C API and add to Sphinx documentation option(DPCTL_ENABLE_DOXYREST "Enable generation of rst files for C API" @@ -55,7 +52,6 @@ function(_setup_doxygen) # Target to generate only Doxygen documentation add_custom_target( Doxygen - ALL DEPENDS ${DOXYGEN_INDEX_FILE} ) endfunction() @@ -95,7 +91,6 @@ function(_setup_doxyrest) # Target to generate rst from Doxygen XML using Doxyrest add_custom_target( Doxyrest - ALL DEPENDS Doxygen ${DOXYREST_OUTPUT} ) endfunction() @@ -147,7 +142,6 @@ function(_setup_sphinx) # important, we want the rst files to generate prior to sphinx build. add_custom_target( Sphinx - ALL DEPENDS ${DEPEND_ON_DOXYREST} ${DPCTL_PYAPI_RST_FILE} @@ -196,12 +190,7 @@ if (DPCTL_ENABLE_DOXYREST) endif() # Set the location where the generated docs are saved -if(DPCTL_DOCGEN_PREFIX) - message(STATUS "Generating dpctl documents in " ${DPCTL_DOCGEN_PREFIX}) - set(DOC_OUTPUT_DIR ${DPCTL_DOCGEN_PREFIX}) -else() - set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/generated_docs) -endif() +set(DOC_OUTPUT_DIR ${CMAKE_INSTALL_PREFIX}/docs) set(INDEX_NO_DOXYREST_IN ${CMAKE_CURRENT_SOURCE_DIR}/index_no_doxyrest.rst.in) set(INDEX_DOXYREST_IN ${CMAKE_CURRENT_SOURCE_DIR}/index_doxyrest.rst.in) diff --git a/scripts/gen_docs.py b/scripts/gen_docs.py new file mode 100644 index 0000000000..9b5435c475 --- /dev/null +++ b/scripts/gen_docs.py @@ -0,0 +1,171 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + bin_llvm=None, + doxyrest_dir=None, +): + IS_LIN = False + + if "linux" in sys.platform: + IS_LIN = True + elif sys.platform in ["win32", "cygwin"]: + pass + else: + assert False, sys.platform + " not supported" + + if not IS_LIN: + raise RuntimeError( + "This scripts only supports coverage collection on Linux" + ) + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + "--", + "-G", + "Unix Makefiles", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + "-DDPCTL_GENERATE_DOCS=ON", + ] + + if doxyrest_dir: + cmake_args.append("-DDPCTL_ENABLE_DOXYREST=ON") + cmake_args.append("-DDoxyrest_DIR=" + doxyrest_dir) + + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + env = None + if bin_llvm: + env = { + "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), + } + env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + # Install dpctl package + subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + # Get the path for the build directory + build_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "cmake-build"], + cwd=setup_dir, + ) + .decode("utf-8") + .strip("\n") + ) + # Generate docs + subprocess.check_call( + ["cmake", "--build", ".", "--target", "Sphinx"], cwd=build_dir + ) + generated_doc_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "index.html"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) + print("Generated documentation placed under ", generated_doc_dir) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl and generate coverage" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--not-oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_false", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + driver.add_argument( + "--bin-llvm", help="Path to folder where llvm-cov can be found" + ) + driver.add_argument( + "--doxyrest-root", + help=( + "Path to Doxyrest installation to use to generate Sphinx docs" + + "for libsyclinterface" + ), + ) + + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" + args.compiler_root = None + icx_path = subprocess.check_output(["which", "icx"]) + bin_dir = os.path.dirname(os.path.dirname(icx_path)) + args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + "bin_llvm", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + bin_llvm=args.bin_llvm, + doxyrest_dir=args.doxyrest_root, + ) From 0455e608b129470f29eb31a0547c22ca619fde29 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 2 Feb 2022 19:38:22 -0600 Subject: [PATCH 205/229] Update REAMDE for docs. --- docs/README.md | 57 ++++++-------------------------------------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/docs/README.md b/docs/README.md index 20e696f1a2..075889c9af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,58 +32,13 @@ sudo apt-get install liblua5.2-dev Generating the docs =================== -The documentation should be generated using the provided `Cmake` build script. -There are a few configurable options that can be used to select the type of -documentation to generate. +The helper script ``scripts/gen_docs.py`` is the preferred way to generate the +documentation. The generated documentation html pages will be installed to the +``CMAKE_INSTALL_PREFIX/docs`` directory. -Build only Doxygen for C API ---------------------------- ```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -make Doxygen +python scripts/gen_docs.py --doxyrest-root= ``` -The above steps will generate the `Doxygen` files at -`dpctl/docs/generated_docs/doxygen/html`. The documentation can also be -generated at a custom location by providing the optional flag - -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -DDPCTL_DOCGEN_PREFIX= -make Doxygen -``` - -Build only Sphinx for Python API --------------------------------- -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -DDPCTL_DOCGEN_PREFIX= -make Sphinx -``` - -The `make Sphinx` command will generate only the Python API docs for dpctl. - -Build consolidated docs ------------------------ -It is possible to generate a single site with both Python and C API docs. As -mentioned before, `Doxyrest` and `Lua` are required to generate the consolidated -site. - -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. \ - -DDPCTL_ENABLE_DOXYREST=ON \ - -DDoxyrest_DIR= \ - -DDPCTL_DOCGEN_PREFIX= -make Sphinx -``` -The `Doxyrest_DIR` flag is optional, but is needed when Doxyrest is installed in -a non-system location. +To skip generating the documentation for ``libsyclinterface``, the +``--doxyrest-root`` option should be omitted. From 52d29afca0d88cc04965fcd4659da0e9de3f6b7e Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 2 Feb 2022 22:35:25 -0600 Subject: [PATCH 206/229] Merge pull request #762 from diptorupd/fix/748 Remove redundant namespace scopes. --- dpctl/apis/include/dpctl4pybind11.hpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp index b2e5e05633..eb563b3f84 100644 --- a/dpctl/apis/include/dpctl4pybind11.hpp +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -67,13 +67,7 @@ template <> struct type_caster return handle(reinterpret_cast(tmp)); } }; -} // namespace detail -} // namespace pybind11 -namespace pybind11 -{ -namespace detail -{ /* This type caster associates ``sycl::device`` C++ class with * :class:`dpctl.SyclDevice` for the purposes of generation of * Python bindings by pybind11. @@ -105,13 +99,7 @@ template <> struct type_caster return handle(reinterpret_cast(tmp)); } }; -} // namespace detail -} // namespace pybind11 -namespace pybind11 -{ -namespace detail -{ /* This type caster associates ``sycl::context`` C++ class with * :class:`dpctl.SyclContext` for the purposes of generation of * Python bindings by pybind11. @@ -144,13 +132,7 @@ template <> struct type_caster return handle(reinterpret_cast(tmp)); } }; -} // namespace detail -} // namespace pybind11 -namespace pybind11 -{ -namespace detail -{ /* This type caster associates ``sycl::event`` C++ class with * :class:`dpctl.SyclEvent` for the purposes of generation of * Python bindings by pybind11. From 0cf78566439c3ff98409d32347a9f679b385ec98 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 09:49:57 -0600 Subject: [PATCH 207/229] Update vendored versioneer from 0.18 to 0.21 (#764) --- .gitattributes | 1 + MANIFEST.in | 1 - dpctl/_version.py | 374 ++++++++++++++---------- versioneer.py | 713 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 732 insertions(+), 357 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..f10261fb12 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +dpctl/_version.py export-subst diff --git a/MANIFEST.in b/MANIFEST.in index 604da428cc..9ace2ea6ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include versioneer.py recursive-include dpctl/include *.h include dpctl/include/dpctl4pybind11.hpp recursive-include dpctl *.pxd diff --git a/dpctl/_version.py b/dpctl/_version.py index 395c950102..d166fbc8b9 100644 --- a/dpctl/_version.py +++ b/dpctl/_version.py @@ -1,3 +1,4 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -5,7 +6,7 @@ # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer) """Git implementation of _version.py.""" @@ -14,6 +15,7 @@ import re import subprocess import sys +from typing import Callable, Dict def get_keywords(): @@ -41,8 +43,8 @@ def get_config(): cfg.VCS = "git" cfg.style = "" cfg.tag_prefix = "" - cfg.parentdir_prefix = "DP-Glue-" - cfg.versionfile_source = "dpglue/_version.py" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "dpctl/_version.py" cfg.verbose = False return cfg @@ -51,40 +53,36 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f - return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -96,15 +94,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): @@ -116,25 +112,18 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -147,22 +136,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -170,10 +158,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -186,11 +178,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +191,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -207,30 +199,28 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] + r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -238,10 +228,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"\*" - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -249,24 +242,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%s%s" % (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -276,6 +261,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -284,16 +302,17 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] + git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + # unparsable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) return pieces # tag @@ -302,12 +321,10 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] + pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -318,13 +335,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -355,25 +373,74 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered @@ -404,12 +471,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -469,23 +565,25 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -495,13 +593,9 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): @@ -515,7 +609,8 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) except NotThisMethod: pass @@ -524,16 +619,13 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -547,10 +639,6 @@ def get_versions(): except NotThisMethod: pass - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/versioneer.py b/versioneer.py index 64fea1c892..b4cd1d6c7c 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.18 +# Version: 0.21 """The Versioneer - like a rocketeer, but for versions. @@ -7,16 +7,12 @@ ============== * like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer +* https://github.com/python-versioneer/python-versioneer * Brian Warner * License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) +* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] This is a tool for managing a recorded version number in distutils-based python projects. The goal is to remove the tedious and error-prone "update @@ -27,9 +23,10 @@ ## Quick Install -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) +* `pip install versioneer` to somewhere in your $PATH +* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) * run `versioneer install` in your source tree, commit the results +* Verify version information with `python setup.py version` ## Version Identifiers @@ -61,7 +58,7 @@ for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. +uncommitted changes). The version identifier is used for multiple purposes: @@ -166,7 +163,7 @@ Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). +[issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects @@ -180,7 +177,7 @@ `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. + provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs @@ -194,9 +191,9 @@ Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve @@ -224,22 +221,10 @@ cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - ## Updating Versioneer @@ -265,6 +250,14 @@ direction and include code from all supported VCS systems, reducing the number of intermediate scripts. +## Similar projects + +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer +* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools + plugin ## License @@ -274,19 +267,27 @@ Dedication" license (CC0-1.0), as described in https://creativecommons.org/publicdomain/zero/1.0/ . +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer + """ +# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring +# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements +# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error +# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with +# pylint:disable=attribute-defined-outside-init,too-many-arguments -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import errno import json import os import re import subprocess import sys +from typing import Callable, Dict class VersioneerConfig: @@ -321,12 +322,12 @@ def get_root(): # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) + my_path = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root @@ -334,30 +335,29 @@ def get_root(): def get_config_from_root(root): """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or + # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) + parser = configparser.ConfigParser() + with open(setup_cfg, "r") as cfg_file: + parser.read_file(cfg_file) VCS = parser.get("versioneer", "VCS") # mandatory - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None + # Dict-like interface for non-mandatory entries + section = parser["versioneer"] + cfg = VersioneerConfig() cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") + cfg.style = section.get("style", "") + cfg.versionfile_source = section.get("versionfile_source") + cfg.versionfile_build = section.get("versionfile_build") + cfg.tag_prefix = section.get("tag_prefix") if cfg.tag_prefix in ("''", '""'): cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") + cfg.parentdir_prefix = section.get("parentdir_prefix") + cfg.verbose = section.get("verbose") return cfg @@ -366,17 +366,15 @@ class NotThisMethod(Exception): # these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f + HANDLERS.setdefault(vcs, {})[method] = f return f return decorate @@ -385,17 +383,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -407,18 +405,16 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -426,7 +422,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer) """Git implementation of _version.py.""" @@ -435,6 +431,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, import re import subprocess import sys +from typing import Callable, Dict def get_keywords(): @@ -472,12 +469,12 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: @@ -491,17 +488,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -513,15 +510,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): @@ -533,15 +528,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% @@ -558,22 +552,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -581,10 +574,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -597,11 +594,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d @@ -610,7 +607,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: @@ -619,6 +616,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %%s" %% r) return {"version": r, @@ -634,7 +636,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -642,11 +644,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"\*" - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) @@ -654,15 +658,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%%s%%s" %% (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -672,6 +677,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -688,7 +726,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces @@ -713,13 +751,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -757,19 +796,67 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%%d.dev%%d" %% (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%%d" %% (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] + rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered @@ -800,12 +887,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -876,10 +992,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -915,7 +1035,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, @@ -950,22 +1070,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -973,10 +1092,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -989,11 +1112,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1002,7 +1125,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,6 +1134,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) return {"version": r, @@ -1026,7 +1154,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -1034,11 +1162,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"\*" - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1046,15 +1176,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%s%s" % (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -1064,6 +1195,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -1080,7 +1244,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces @@ -1105,13 +1269,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1130,27 +1295,26 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): if ipy: files.append(ipy) try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) + my_path = __file__ + if my_path.endswith(".pyc") or my_path.endswith(".pyo"): + my_path = os.path.splitext(my_path)[0] + ".py" + versioneer_file = os.path.relpath(my_path) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) present = False try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: + with open(".gitattributes", "r") as fobj: + for line in fobj: + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + break + except OSError: pass if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() + with open(".gitattributes", "a+") as fobj: + fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) @@ -1164,15 +1328,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % @@ -1181,7 +1344,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from +# This file was generated by 'versioneer.py' (0.21) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. @@ -1203,7 +1366,7 @@ def versions_from_file(filename): try: with open(filename) as f: contents = f.read() - except EnvironmentError: + except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) @@ -1258,19 +1421,67 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered @@ -1301,12 +1512,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -1377,10 +1617,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -1480,8 +1724,12 @@ def get_version(): return get_versions()["version"] -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" +def get_cmdclass(cmdclass=None): + """Get the custom setuptools/distutils subclasses used by Versioneer. + + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and @@ -1495,9 +1743,9 @@ def get_cmdclass(): # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - cmds = {} + cmds = {} if cmdclass is None else cmdclass.copy() # we add "version" to both distutils and setuptools from distutils.core import Command @@ -1539,7 +1787,9 @@ def run(self): # setup.py egg_info -> ? # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: + if 'build_py' in cmds: + _build_py = cmds['build_py'] + elif "setuptools" in sys.modules: from setuptools.command.build_py import build_py as _build_py else: from distutils.command.build_py import build_py as _build_py @@ -1559,6 +1809,33 @@ def run(self): write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py + if 'build_ext' in cmds: + _build_ext = cmds['build_ext'] + elif "setuptools" in sys.modules: + from setuptools.command.build_ext import build_ext as _build_ext + else: + from distutils.command.build_ext import build_ext as _build_ext + + class cmd_build_ext(_build_ext): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_ext.run(self) + if self.inplace: + # build_ext --inplace will only build extensions in + # build/lib<..> dir with no _version.py to write to. + # As in place builds will already have a _version.py + # in the module dir, we do not need to write one. + return + # now locate _version.py in the new build/ directory and replace + # it with an updated value + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext + if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # nczeczulin reports that py2exe won't like the pep440-style string @@ -1592,10 +1869,7 @@ def run(self): del cmds["build_py"] if 'py2exe' in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 + from py2exe.distutils_buildexe import py2exe as _py2exe class cmd_py2exe(_py2exe): def run(self): @@ -1620,7 +1894,9 @@ def run(self): cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: + if 'sdist' in cmds: + _sdist = cmds['sdist'] + elif "setuptools" in sys.modules: from setuptools.command.sdist import sdist as _sdist else: from distutils.command.sdist import sdist as _sdist @@ -1687,21 +1963,26 @@ def make_release_tree(self, base_dir, files): """ -INIT_PY_SNIPPET = """ +OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ +INIT_PY_SNIPPET = """ +from . import {0} +__version__ = {0}.get_versions()['version'] +""" + def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" + """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: @@ -1725,12 +2006,18 @@ def do_setup(): try: with open(ipy, "r") as f: old = f.read() - except EnvironmentError: + except OSError: old = "" - if INIT_PY_SNIPPET not in old: + module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] + snippet = INIT_PY_SNIPPET.format(module) + if OLD_SNIPPET in old: + print(" replacing boilerplate in %s" % ipy) + with open(ipy, "w") as f: + f.write(old.replace(OLD_SNIPPET, snippet)) + elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) + f.write(snippet) else: print(" %s unmodified" % ipy) else: @@ -1749,7 +2036,7 @@ def do_setup(): if line.startswith("include "): for include in line.split()[1:]: simple_includes.add(include) - except EnvironmentError: + except OSError: pass # That doesn't cover everything MANIFEST.in can do # (http://docs.python.org/2/distutils/sourcedist.html#commands), so From 3b3979ef3c136da614818ee27d9a2096ac2c3f12 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Wed, 2 Feb 2022 20:10:56 -0600 Subject: [PATCH 208/229] Update the documentation generation github workflow to use new infrastructure. --- .github/workflows/generate-docs.yml | 31 +++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 052a82a304..39bbf2e26a 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -55,31 +55,28 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Build dpctl - if: ${{ !github.event.pull_request || github.event.action != 'closed' }} - shell: bash -l {0} - run: | - source /opt/intel/oneapi/setvars.sh - python setup.py develop -- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX} - python -c "import dpctl; print(dpctl.__version__)" || exit 1 - - name: Build docs + - name: Build dpctl+docs if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | # Ensure that SYCL libraries are on LD_LIBRARY_PATH source /opt/intel/oneapi/setvars.sh - cd docs - mkdir -p build && cd build && rm -rf * wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-2.1.2/doxyrest-2.1.2-linux-amd64.tar.xz tar xf doxyrest-2.1.2-linux-amd64.tar.xz - cmake .. -DDPCTL_USE_MULTIVERSION_TEMPLATE=ON \ - -DDPCTL_ENABLE_DOXYREST=ON \ - -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 - make Sphinx || exit 1 - cd .. - mv generated_docs/docs ~/docs + python setup.py develop -- \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER:PATH=icx \ + -DCMAKE_CXX_COMPILER:PATH=icpx \ + -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON \ + -DDPCTL_GENERATE_DOCS=ON \ + -DDPCTL_ENABLE_DOXYREST=ON \ + -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 + python -c "import dpctl; print(dpctl.__version__)" || exit 1 + cd "$(find _skbuild -name cmake-build)" || exit 1 + cmake --build . --target Sphinx || exit 1 + mv ../cmake-install/docs/docs ~/docs git clean -dfx - cd .. - name: Publish docs if: ${{ github.ref == 'refs/heads/master' }} shell: bash -l {0} From 35af9f2c258303162644d57590b3248f0f2b12ff Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 14:02:18 -0600 Subject: [PATCH 209/229] Use -t flag for Cython to reuse CXX files is PYX did not change --- dpctl/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 3ca3c9a79d..14f9b7d38a 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -2,7 +2,7 @@ find_package(PythonExtensions REQUIRED) find_package(NumPy REQUIRED) -set(CYTHON_FLAGS "-w ${CMAKE_SOURCE_DIR}") +set(CYTHON_FLAGS "-t -w ${CMAKE_SOURCE_DIR}") find_package(Cython REQUIRED) if(WIN32) From 5e26e5d76eb26bce02f874328a68d0b64620b1d0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 14:02:56 -0600 Subject: [PATCH 210/229] Drop use of COMPONENT in install commands --- CMakeLists.txt | 1 - libsyclinterface/CMakeLists.txt | 2 -- 2 files changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8aff7d1d70..858c83c5fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,6 @@ add_subdirectory(libsyclinterface) file(GLOB _dpctl_capi_headers dpctl/apis/include/*.h*) install(FILES ${_dpctl_capi_headers} DESTINATION dpctl/include - COMPONENT DpctlCAPIHeaders ) add_subdirectory(dpctl) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index a6c73f3878..e63c992acb 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -241,12 +241,10 @@ install(TARGETS install( FILES ${SUPPORT_HEADERS} DESTINATION ${_include_destination}/Support - COMPONENT SupportHeaders ) install( FILES ${CONFIG_HEADERS} DESTINATION ${_include_destination}/Config - COMPONENT ConfigHeaders ) # Enable code coverage related settings From db6b9533973707c33b0cc7187c1bc9131da6ed32 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 15:49:38 -0600 Subject: [PATCH 211/229] Include all headers in PUBLIC_HEADER property of the library --- libsyclinterface/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index e63c992acb..6beed813a8 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -214,9 +214,13 @@ file(GLOB MAIN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") file(GLOB SUPPORT_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") file(GLOB CONFIG_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") +set(ALL_HEADERS "${MAIN_HEADERS}") +list(APPEND ALL_HEADERS "${SUPPORT_HEADERS}") +list(APPEND ALL_HEADERS "${CONFIG_HEADERS}") + set_target_properties(DPCTLSyclInterface PROPERTIES PUBLIC_HEADER - "${MAIN_HEADERS}" + "${ALL_HEADERS}" ) if (SKBUILD) From dff9c439c2e976f26a84522068d71118903aef1a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 19:08:20 -0600 Subject: [PATCH 212/229] Change the CMake script that for copying header files to dpctl/include Instead of using add_custom_command for a TARGET which does not specify the required PRE_BUILD/PRE_LINK/POST_LINK argument, use add_custom_command with OUTPUT to perform the copy, record files coped and create a custom target that depend on those. All Cython files then depend on that target. --- dpctl/CMakeLists.txt | 49 ++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt index 14f9b7d38a..541ef7d5e8 100644 --- a/dpctl/CMakeLists.txt +++ b/dpctl/CMakeLists.txt @@ -78,41 +78,64 @@ add_custom_target(_build_time_create_dpctl_include ALL DEPENDS DPCTLSyclInterface ) +set(_copied_header_files) file(GLOB _syclinterface_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/*.h) foreach(hf ${_syclinterface_h}) - add_custom_command(TARGET _build_time_create_dpctl_include - COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface - ) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + VERBATIM + ) endforeach() file(GLOB _syclinterface_Support_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Support/*.h) foreach(hf ${_syclinterface_Support_h}) - add_custom_command(TARGET _build_time_create_dpctl_include - COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface/Support - ) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/Support/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) endforeach() file(GLOB _syclinterface_Config_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Config/*.h) foreach(hf ${_syclinterface_Config_h}) - add_custom_command(TARGET _build_time_create_dpctl_include - COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR}/syclinterface/Config - ) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/Config/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) endforeach() file(GLOB _apis_h ${CMAKE_CURRENT_SOURCE_DIR}/apis/include/*) foreach(hf ${_apis_h}) - add_custom_command(TARGET _build_time_create_dpctl_include - COMMAND ${CMAKE_COMMAND} -E copy ${hf} ${DPCTL_INCLUDE_DIR} - ) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) endforeach() +add_custom_target( + _build_time_create_dpctl_include_copy ALL + DEPENDS ${_copied_header_files} +) + set(CMAKE_INSTALL_RPATH "$ORIGIN") function(build_dpctl_ext _trgt _src _dest) add_cython_target(${_trgt} ${_src} CXX OUTPUT_VAR _generated_src) add_library(${_trgt} MODULE ${_generated_src}) target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) - add_dependencies(${_trgt} _build_time_create_dpctl_include) + add_dependencies(${_trgt} _build_time_create_dpctl_include_copy) if (DPCTL_GENERATE_COVERAGE) target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1) target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer) From 524c3da1964b5205368b10d2d5bad6315dfd3712 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 19:13:10 -0600 Subject: [PATCH 213/229] Use Ninja generated for Linux as well --- scripts/build_locally.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 19b811f376..acd1a4a6a0 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -32,7 +32,7 @@ def run( build_system = None if "linux" in sys.platform: - build_system = "Unix Makefiles" + build_system = "Ninja" elif sys.platform in ["win32", "cygwin"]: build_system = "Ninja" else: From acb158ef69f5763cbe1c0d5d354933361c765415 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 3 Feb 2022 19:21:37 -0600 Subject: [PATCH 214/229] Use Ninja generator on both Linux and Windows Ninja parallalizes the build and hence works faster to build the project, it also does not have the bug that make has cause it to unconditionally recompile and relink extension libraries saving significant iterative development time. --- .github/workflows/generate-coverage.yaml | 10 ++++++---- .github/workflows/generate-docs.yml | 6 +++--- .github/workflows/os-llvm-sycl-build.yml | 4 ++-- conda-recipe/build.sh | 2 +- conda-recipe/meta.yaml | 3 +-- docs/docfiles/user_guides/QuickStart.rst | 10 +++++----- scripts/gen_coverage.py | 2 +- scripts/gen_docs.py | 2 +- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index d21da02645..89234307b3 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -32,9 +32,9 @@ jobs: sudo apt-get install intel-oneapi-compiler-dpcpp-cpp sudo apt-get install intel-oneapi-tbb - - name: Install CMake + - name: Install CMake and Ninja run: | - sudo apt-get install cmake + sudo apt-get install cmake ninja-build - name: Setup Python uses: actions/setup-python@v2 @@ -87,8 +87,10 @@ jobs: source /opt/intel/oneapi/setvars.sh export _SAVED_PATH=${PATH} export PATH=$(dirname $(dirname $(which icx)))/bin-llvm:${PATH} - python setup.py develop -- -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_GENERATE_COVERAGE=ON -DDPCTL_BUILD_CAPI_TESTS=ON -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) - make -C $(find _skbuild -name tests) lcov-genhtml + python setup.py develop -- -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_GENERATE_COVERAGE=ON -DDPCTL_BUILD_CAPI_TESTS=ON -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) + pushd $(find _skbuild -name tests) + cmake --build . --target lcov-genhtml || exit 1 + popd export PATH=${_SAVED_PATH} unset _SAVED_PATH python -c "import dpctl; print(dpctl.__version__); dpctl.lsplatform()" || exit 1 diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 39bbf2e26a..adf8717192 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -35,10 +35,10 @@ jobs: if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install doxygen - - name: Install CMake + - name: Install CMake and Ninja if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | - sudo apt-get install cmake + sudo apt-get install cmake ninja-build - name: Setup Python if: ${{ !github.event.pull_request || github.event.action != 'closed' }} uses: actions/setup-python@v2 @@ -64,7 +64,7 @@ jobs: wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-2.1.2/doxyrest-2.1.2-linux-amd64.tar.xz tar xf doxyrest-2.1.2-linux-amd64.tar.xz python setup.py develop -- \ - -G "Unix Makefiles" \ + -G "Ninja" \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_COMPILER:PATH=icx \ -DCMAKE_CXX_COMPILER:PATH=icpx \ diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index 8268bd0683..c1c0329db7 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -68,7 +68,7 @@ jobs: - name: Install system components shell: bash -l {0} run: | - sudo apt-get install cmake libtinfo5 + sudo apt-get install cmake ninja-build libtinfo5 - name: Setup Python uses: actions/setup-python@v2 @@ -97,6 +97,6 @@ jobs: export OCL_ICD_FILENAMES=libintelocl.so:libintelocl_emu.so clang++ --version sycl-ls - python setup.py develop -- -G "Unix Makefiles" -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=$(dirname $(dirname $(which clang))) -DDPCTL_DPCPP_FROM_ONEAPI=OFF + python setup.py develop -- -G Ninja -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=$(dirname $(dirname $(which clang))) -DDPCTL_DPCPP_FROM_ONEAPI=OFF python -c "import dpctl; dpctl.lsplatform()" || exit 1 SYCL_ENABLE_HOST_DEVICE=1 python -m pytest -v dpctl/tests diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 9d581001fe..5bc85e861e 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -6,7 +6,7 @@ export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib" ${PYTHON} setup.py clean --all -export CMAKE_GENERATOR="Unix Makefiles" +export CMAKE_GENERATOR="Ninja" SKBUILD_ARGS="-- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=${BUILD_PREFIX}" echo "${PYTHON} setup.py install ${SKBUILD_ARGS}" diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index c3087c303a..27322b0030 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -20,8 +20,7 @@ requirements: - cython - cmake >=3.21 - python - - make # [unix] - - ninja # [win] + - ninja - scikit-build - numpy 1.19 - wheel diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index 8c58acd15a..94b4953ca3 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -143,7 +143,7 @@ installed: - ``numpy`` - ``cmake`` - ``scikit-build`` - - ``ninja`` (only on Windows) + - ``ninja`` - ``gtest`` (optional to run C API tests) - ``gmock`` (optional to run C API tests) - ``pytest`` (optional to run Python API tests) @@ -152,15 +152,15 @@ Once the prerequisites are installed, building using ``scikit-build`` involves t .. code-block:: bash - python setup.py install -- -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON + python setup.py install -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON , and to develop: .. code-block:: bash - python setup.py develop -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON + python setup.py develop -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -On Windows, use ``icx`` for both C and CXX compilers, and use :code:`-G Ninja` for cmake generator. +On Windows, use ``icx`` for both C and CXX compilers. Developing on Linux can also be done using driver script: @@ -180,7 +180,7 @@ Following steps in `Build and install with scikit-build`_ use command line optio .. code-block:: bash - python setup.py develop -- -G Unix\ Makefiles -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ONE -DDPCTL_DPCPP_HOME_DIR=${DPCPP_ROOT}/llvm/build -DDPCTL_DPCPP_FROM_ONEAPI=OFF + python setup.py develop -- -G Ninja -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ONE -DDPCTL_DPCPP_HOME_DIR=${DPCPP_ROOT}/llvm/build -DDPCTL_DPCPP_FROM_ONEAPI=OFF Alterantively, the driver script can be used diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 43cffd1cd9..2e62727587 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -48,7 +48,7 @@ def run( "develop", "--", "-G", - "Unix Makefiles", + "Ninja", "-DCMAKE_BUILD_TYPE=Debug", "-DCMAKE_C_COMPILER:PATH=" + c_compiler, "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, diff --git a/scripts/gen_docs.py b/scripts/gen_docs.py index 9b5435c475..087bb32230 100644 --- a/scripts/gen_docs.py +++ b/scripts/gen_docs.py @@ -48,7 +48,7 @@ def run( "develop", "--", "-G", - "Unix Makefiles", + "Ninja", "-DCMAKE_BUILD_TYPE=Debug", "-DCMAKE_C_COMPILER:PATH=" + c_compiler, "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, From 8311e08beaa2b911359f8a390dce95a10bbaa117 Mon Sep 17 00:00:00 2001 From: Diptorup Deb Date: Fri, 4 Feb 2022 01:21:57 -0600 Subject: [PATCH 215/229] Improvements to gen_coverage.py helper script. -- Add an option to specify custom gtest installation. -- FindLLVMCov offers a way to specify where to look for llvm-cov. If `bin-llvm` is provided, set the `LLVM_TOOL_DIR` envar as well to help FindLLVMCov. Otherwise, just adding to PATH is not enough. -- Some formatting fixes to coverage workflow. --- .github/workflows/generate-coverage.yaml | 10 +++++++++- scripts/gen_coverage.py | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index 89234307b3..8ef77e3436 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -87,7 +87,15 @@ jobs: source /opt/intel/oneapi/setvars.sh export _SAVED_PATH=${PATH} export PATH=$(dirname $(dirname $(which icx)))/bin-llvm:${PATH} - python setup.py develop -- -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_GENERATE_COVERAGE=ON -DDPCTL_BUILD_CAPI_TESTS=ON -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) + python setup.py develop -- \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER:PATH=icx \ + -DCMAKE_CXX_COMPILER:PATH=icpx \ + -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON \ + -DDPCTL_GENERATE_COVERAGE=ON \ + -DDPCTL_BUILD_CAPI_TESTS=ON \ + -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) pushd $(find _skbuild -name tests) cmake --build . --target lcov-genhtml || exit 1 popd diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 2e62727587..7f8b357bae 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -27,6 +27,7 @@ def run( compiler_root=None, run_pytest=False, bin_llvm=None, + gtest_config=None, ): IS_LIN = False @@ -66,18 +67,14 @@ def run( if bin_llvm: env = { "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), + "LLVM_TOOLS_HOME": bin_llvm, } env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + if gtest_config: + cmake_args += ["-DCMAKE_PREFIX_PATH=" + gtest_config] subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) - test_dir = ( - subprocess.check_output( - ["find", "_skbuild", "-name", "tests"], cwd=setup_dir - ) - .decode("utf-8") - .strip("\n") - ) subprocess.check_call( - ["make", "-C", test_dir, "lcov-genhtml"], cwd=setup_dir + ["cmake", "--build", ".", "--target", "lcov-genhtml"], cwd=setup_dir ) subprocess.check_call( [ @@ -135,6 +132,11 @@ def run( driver.add_argument( "--bin-llvm", help="Path to folder where llvm-cov can be found" ) + driver.add_argument( + "--gtest-config", + help="Path to the GTestConfig.cmake file to locate a " + + "custom GTest installation.", + ) args = parser.parse_args() if args.oneapi: @@ -170,4 +172,5 @@ def run( compiler_root=args.compiler_root, run_pytest=args.run_pytest, bin_llvm=args.bin_llvm, + gtest_config=args.gtest_config, ) From 15477c66b02b3045b1328a87915d3b071e3f2370 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 4 Feb 2022 05:04:44 -0600 Subject: [PATCH 216/229] Use DEPENDS arg of add_custom_target rather than CMAKE_MAKE_EXECUTABLE to implement dependency --- libsyclinterface/tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libsyclinterface/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt index dc180ec70d..36037b1d4b 100644 --- a/libsyclinterface/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -61,7 +61,6 @@ if(DPCTL_GENERATE_COVERAGE) ${CMAKE_DL_LIBS} ) add_custom_target(llvm-cov - COMMAND ${CMAKE_MAKE_PROGRAM} dpctl_c_api_tests COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=warning ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests COMMAND ${LLVMProfdata_EXE} merge @@ -75,10 +74,10 @@ if(DPCTL_GENERATE_COVERAGE) ${dpctl_sources} ${helper_sources} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS dpctl_c_api_tests ) add_custom_target(lcov-genhtml - COMMAND ${CMAKE_MAKE_PROGRAM} llvm-cov COMMAND ${LLVMCov_EXE} export -format=lcov @@ -91,6 +90,7 @@ if(DPCTL_GENERATE_COVERAGE) --output-directory ${COVERAGE_OUTPUT_DIR}/dpctl-c-api-coverage WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS llvm-cov ) else() file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) From c24d096f409bd221357eb0973cd4937ec0f51fd9 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 4 Feb 2022 05:05:55 -0600 Subject: [PATCH 217/229] cmake --build . --target lcov-genhtml must be run from cmake-build folder --- .github/workflows/generate-coverage.yaml | 2 +- scripts/gen_coverage.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index 8ef77e3436..b03c9fd031 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -96,7 +96,7 @@ jobs: -DDPCTL_GENERATE_COVERAGE=ON \ -DDPCTL_BUILD_CAPI_TESTS=ON \ -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) - pushd $(find _skbuild -name tests) + pushd $(find _skbuild -name cmake-build) cmake --build . --target lcov-genhtml || exit 1 popd export PATH=${_SAVED_PATH} diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 7f8b357bae..611327689a 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -73,8 +73,16 @@ def run( if gtest_config: cmake_args += ["-DCMAKE_PREFIX_PATH=" + gtest_config] subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + cmake_build_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "cmake-build"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) subprocess.check_call( - ["cmake", "--build", ".", "--target", "lcov-genhtml"], cwd=setup_dir + ["cmake", "--build", ".", "--target", "lcov-genhtml"], + cwd=cmake_build_dir, ) subprocess.check_call( [ From 55056575f7eb0ffaa7bace30781181e49885ce01 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 4 Feb 2022 05:40:20 -0600 Subject: [PATCH 218/229] no need to pin coverage at 3.2.0 (too old) --- .github/workflows/generate-coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index b03c9fd031..c46af799a8 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -108,7 +108,7 @@ jobs: shell: bash -l {0} run: | sudo gem install coveralls-lcov - pip install coveralls==3.2.0 + pip install coveralls - name: Upload coverage data to coveralls.io shell: bash -l {0} From fb6b54ee238a4f65c988f331dc823d8c10124139 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 4 Feb 2022 13:41:50 -0600 Subject: [PATCH 219/229] Merge pull request #768 from IntelPython/cmake-cleanup-2 Fix include/syclinterface structure --- libsyclinterface/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libsyclinterface/CMakeLists.txt b/libsyclinterface/CMakeLists.txt index 6beed813a8..e63c992acb 100644 --- a/libsyclinterface/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -214,13 +214,9 @@ file(GLOB MAIN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") file(GLOB SUPPORT_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") file(GLOB CONFIG_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") -set(ALL_HEADERS "${MAIN_HEADERS}") -list(APPEND ALL_HEADERS "${SUPPORT_HEADERS}") -list(APPEND ALL_HEADERS "${CONFIG_HEADERS}") - set_target_properties(DPCTLSyclInterface PROPERTIES PUBLIC_HEADER - "${ALL_HEADERS}" + "${MAIN_HEADERS}" ) if (SKBUILD) From ae7a4458475fb091641526d755a403e6a494e419 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sat, 5 Feb 2022 17:46:49 -0600 Subject: [PATCH 220/229] usm_ndarray API extended by GetOffset function (#769) also make type constants and flags API symbols as well, so that pybind11 extensions can use them. --- dpctl/tensor/_usmarray.pxd | 42 ++++++++++++++-------------- dpctl/tensor/_usmarray.pyx | 6 ++++ dpctl/tests/test_usm_ndarray_ctor.py | 13 +++++++++ 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/dpctl/tensor/_usmarray.pxd b/dpctl/tensor/_usmarray.pxd index e0d53b05af..4a7997fa94 100644 --- a/dpctl/tensor/_usmarray.pxd +++ b/dpctl/tensor/_usmarray.pxd @@ -4,27 +4,27 @@ cimport dpctl -cdef public int USM_ARRAY_C_CONTIGUOUS -cdef public int USM_ARRAY_F_CONTIGUOUS -cdef public int USM_ARRAY_WRITEABLE - -cdef public int UAR_BOOL -cdef public int UAR_BYTE -cdef public int UAR_UBYTE -cdef public int UAR_SHORT -cdef public int UAR_USHORT -cdef public int UAR_INT -cdef public int UAR_UINT -cdef public int UAR_LONG -cdef public int UAR_ULONG -cdef public int UAR_LONGLONG -cdef public int UAR_ULONGLONG -cdef public int UAR_FLOAT -cdef public int UAR_DOUBLE -cdef public int UAR_CFLOAT -cdef public int UAR_CDOUBLE -cdef public int UAR_TYPE_SENTINEL -cdef public int UAR_HALF +cdef public api int USM_ARRAY_C_CONTIGUOUS +cdef public api int USM_ARRAY_F_CONTIGUOUS +cdef public api int USM_ARRAY_WRITEABLE + +cdef public api int UAR_BOOL +cdef public api int UAR_BYTE +cdef public api int UAR_UBYTE +cdef public api int UAR_SHORT +cdef public api int UAR_USHORT +cdef public api int UAR_INT +cdef public api int UAR_UINT +cdef public api int UAR_LONG +cdef public api int UAR_ULONG +cdef public api int UAR_LONGLONG +cdef public api int UAR_ULONGLONG +cdef public api int UAR_FLOAT +cdef public api int UAR_DOUBLE +cdef public api int UAR_CFLOAT +cdef public api int UAR_CDOUBLE +cdef public api int UAR_TYPE_SENTINEL +cdef public api int UAR_HALF cdef api class usm_ndarray [object PyUSMArrayObject, type PyUSMArrayType]: diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 3d6d16c5b5..6fd97ebbd8 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -1174,3 +1174,9 @@ cdef api int UsmNDArray_GetFlags(usm_ndarray arr): cdef api c_dpctl.DPCTLSyclQueueRef UsmNDArray_GetQueueRef(usm_ndarray arr): """Get DPCTLSyclQueueRef for queue associated with the array""" return arr.get_queue_ref() + + +cdef api Py_ssize_t UsmNDArray_GetOffset(usm_ndarray arr): + """Get offset of zero-index array element from the beginning of the USM + allocation.""" + return arr.get_offset() diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index c9e22ece41..c1c39633d5 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -427,6 +427,19 @@ def test_pyx_capi_get_flags(): assert type(flags) is int and flags == X.flags +def test_pyx_capi_get_offset(): + X = dpt.usm_ndarray(17)[1::2] + get_offset_fn = _pyx_capi_fnptr_to_callable( + X, + "UsmNDArray_GetOffset", + b"Py_ssize_t (struct PyUSMArrayObject *)", + fn_restype=ctypes.c_longlong, + ) + offset = get_offset_fn(X) + assert type(offset) is int + assert offset == X.__sycl_usm_array_interface__["offset"] + + def test_pyx_capi_get_queue_ref(): X = dpt.usm_ndarray(17)[1::2] get_queue_ref_fn = _pyx_capi_fnptr_to_callable( From 5166e25f935c2e1f81b46410ec88b249ee541a4f Mon Sep 17 00:00:00 2001 From: Kirill Yashin Date: Tue, 8 Feb 2022 13:42:47 +0300 Subject: [PATCH 221/229] rebuild (#774) From a2b0b881306be7d31c32bddc95ef69b537305129 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 8 Feb 2022 05:48:59 -0600 Subject: [PATCH 222/229] pybind11 type_caster for queue/device convert Py_None to default constructed instances (#773) --- dpctl/apis/include/dpctl4pybind11.hpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp index eb563b3f84..80d01af721 100644 --- a/dpctl/apis/include/dpctl4pybind11.hpp +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -55,8 +55,12 @@ template <> struct type_caster value = *q; return true; } + else if (source == Py_None) { + value = sycl::queue{}; + return true; + } else { - throw std::runtime_error( + throw py::type_error( "Input is of unexpected type, expected dpctl.SyclQueue"); } } @@ -87,8 +91,12 @@ template <> struct type_caster value = *d; return true; } + else if (source == Py_None) { + value = sycl::device{}; + return true; + } else { - throw std::runtime_error( + throw py::type_error( "Input is of unexpected type, expected dpctl.SyclDevice"); } } @@ -120,7 +128,7 @@ template <> struct type_caster return true; } else { - throw std::runtime_error( + throw py::type_error( "Input is of unexpected type, expected dpctl.SyclContext"); } } @@ -153,7 +161,7 @@ template <> struct type_caster return true; } else { - throw std::runtime_error( + throw py::type_error( "Input is of unexpected type, expected dpctl.SyclEvent"); } } From 03864d00d91e9b010d0b3ea3a8fc301755f88135 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 8 Feb 2022 05:58:07 -0600 Subject: [PATCH 223/229] Fixed type in utility `_memory_overlap`. (#770) To intervals [B1..E1) and [B2..E2) do not overlap when E1 <= B2 || E2 <= B1 <=> B1 >= E2 || B2 >= E1. Hence the overlap may occur is (B1 < E2) and (B2 < E1). --- dpctl/tensor/_copy_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index 87c7179dcc..cb3d434689 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -75,7 +75,8 @@ def _has_memory_overlap(x1, x2): p1_end = p1_beg + m1.nbytes p2_beg = m2._pointer p2_end = p2_beg + m2.nbytes - return p1_beg > p2_end or p2_beg < p1_end + # may intersect if not ((p1_beg >= p2_end) or (p2_beg >= p2_end)) + return (p1_beg < p2_end) and (p2_beg < p1_end) else: return False else: From e5789bfaa8067ddc42f21ad59bcba0aebee66d19 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 8 Feb 2022 08:40:15 -0600 Subject: [PATCH 224/229] Merge pull request #775 from IntelPython/typo-extras_require fixed typo in setup call: extra_require->extras_require --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c660f01180..d043499c5a 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ def _patched_copy_file(src_file, dest_file, hide_listing=True): install_requires=[ "numpy", ], - extra_require={ + extras_require={ "docs": [ "Cython", "sphinx", From 902fa016c6ba4efa8c27dd0ce27b25add94d2aa8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 10 Feb 2022 19:40:00 -0600 Subject: [PATCH 225/229] Execute body of host_task if main interpreter is still initialized (#776) --- dpctl/_host_task_util.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dpctl/_host_task_util.hpp b/dpctl/_host_task_util.hpp index 5208e11d2c..7871cdebfc 100644 --- a/dpctl/_host_task_util.hpp +++ b/dpctl/_host_task_util.hpp @@ -55,7 +55,8 @@ int async_dec_ref(DPCTLSyclQueueRef QRef, *(reinterpret_cast(ERefs[ev_id]))); } cgh.host_task([obj_array_size, obj_vec]() { - { + // if the main thread has not finilized the interpreter yet + if (Py_IsInitialized()) { PyGILState_STATE gstate; gstate = PyGILState_Ensure(); for (size_t i = 0; i < obj_array_size; ++i) { From f7acaeaa929f6f783535d2d8ac18872d3519ef24 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Feb 2022 11:02:24 -0600 Subject: [PATCH 226/229] Closes #777 (#778) When scikit-build assembles dpctl, it runs cmake with "_skbuild/*/cmake-build" being its build directory, "_skbuild/*/cmake-install" being its install directory. It then runs setuptools.setup to execute build_py steps using "_skbuild/*/setuptools" as its build_base. Since sycl interface library is registered in package-data, it is copied from cmake-install (where is has symbolic links) to setuptools (where unpatched setuptools.command.build_py command follows them turning them into hard files). Then, when running install_lib step, files from _"skbuild/*/setuptools/lib/dpctl" are copied into site-packages/dpctl using copy_tree, which also follows symbolic links, hence turning them into hard links). This PR transfers logic from pre-scikit-build setup.py to fix hard links as post setuptools.command.install.run() step). A test is added to tests/test_service.py to verify that on Linux some of library files are symbolic links. --- dpctl/tests/test_service.py | 23 ++++++++++++++++++ setup.py | 47 ++++++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index 53f895d20b..291067b152 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -23,6 +23,7 @@ import os import os.path import re +import sys import pytest @@ -129,3 +130,25 @@ def test_dev_utils(): with pytest.raises(ValueError): with ctx_mngr(log_dir="/not_a_dir"): dpctl.SyclDevice().parent_device + + +def test_syclinterface(): + install_dir = os.path.dirname(os.path.abspath(dpctl.__file__)) + paths = glob.glob(os.path.join(install_dir, "*DPCTLSyclInterface*")) + if "linux" in sys.platform: + assert len(paths) > 1 and any( + [os.path.islink(fn) for fn in paths] + ), "All library instances are hard links" + elif sys.platform in ["win32", "cygwin"]: + exts = [] + for fn in paths: + _, file_ext = os.path.splitext(fn) + exts.append(file_ext.lower()) + assert ( + ".lib" in exts + ), "Installation does not have DPCTLSyclInterface.lib" + assert ( + ".dll" in exts + ), "Installation does not have DPCTLSyclInterface.dll" + else: + raise RuntimeError("Unsupported system") diff --git a/setup.py b/setup.py index d043499c5a..9a26a8f211 100644 --- a/setup.py +++ b/setup.py @@ -14,14 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import os.path import pathlib import shutil +import sys import skbuild import skbuild.setuptools_wrap import skbuild.utils from setuptools import find_packages +from skbuild.command.build_py import build_py as _skbuild_build_py +from skbuild.command.install import install as _skbuild_install import versioneer @@ -30,11 +34,6 @@ long_description = file.read() -def _get_cmdclass(): - cmdclass = versioneer.get_cmdclass() - return cmdclass - - def cleanup_destination(cmake_manifest): """Delete library files from dpctl/ folder before letting skbuild copy them over to avoid errors. @@ -52,7 +51,9 @@ def cleanup_destination(cmake_manifest): return cmake_manifest -def _patched_copy_file(src_file, dest_file, hide_listing=True): +def _patched_copy_file( + src_file, dest_file, hide_listing=True, preserve_mode=True +): """Copy ``src_file`` to ``dest_file`` ensuring parent directory exists. By default, message like `creating directory /path/to/package` and @@ -79,6 +80,40 @@ def _patched_copy_file(src_file, dest_file, hide_listing=True): skbuild.setuptools_wrap._copy_file = _patched_copy_file +class BuildPyCmd(_skbuild_build_py): + def copy_file(self, src, dst, preserve_mode=True): + _patched_copy_file(src, dst, preserve_mode=preserve_mode) + return (dst, 1) + + +class InstallCmd(_skbuild_install): + def run(self): + ret = super().run() + if "linux" in sys.platform: + this_dir = os.path.dirname(os.path.abspath(__file__)) + dpctl_build_dir = os.path.join(this_dir, self.build_lib, "dpctl") + dpctl_install_dir = os.path.join(self.install_libbase, "dpctl") + for fn in glob.glob( + os.path.join(dpctl_install_dir, "*DPCTLSyclInterface.so*") + ): + os.remove(fn) + base_fn = os.path.basename(fn) + src_file = os.path.join(dpctl_build_dir, base_fn) + dst_file = os.path.join(dpctl_install_dir, base_fn) + _patched_copy_file(src_file, dst_file) + return ret + + +def _get_cmdclass(): + cmdclass = versioneer.get_cmdclass( + cmdclass={ + "build_py": BuildPyCmd, + "install": InstallCmd, + } + ) + return cmdclass + + skbuild.setup( name="dpctl", version=versioneer.get_version(), From 335fb238ae8ce17b8abb0731c5c17f027e739cb3 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 24 Feb 2022 17:33:24 -0600 Subject: [PATCH 227/229] operator.pos(ums_ndarray) is always an identity (#783) Since dpnp.positive is not implemented, to unblock integration testing implemented __pos__ to return self. --- dpctl/tensor/_usmarray.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 6fd97ebbd8..b1ce20a1b2 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -857,7 +857,7 @@ cdef class usm_ndarray: return NotImplemented def __pos__(self): - return _dispatch_unary_elementwise(self, "positive") + return self # _dispatch_unary_elementwise(self, "positive") def __pow__(first, other, mod): "See comment in __add__" From 537437d06b3f908f2f4c46af5ad85d90716434b1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 28 Feb 2022 11:53:14 -0600 Subject: [PATCH 228/229] versioneer 0.20 regression fixed Versioneer 0.20 added escape character to the match option of git describe ``` (base) C:\Users\opavlyk\devel\dpctl>git describe --tags --dirty --always --long --match "\*" 3bd37749 (base) C:\Users\opavlyk\devel\dpctl>git describe --tags --dirty --always --long --match "\\*" 3bd37749 (base) C:\Users\opavlyk\devel\dpctl>git describe --tags --dirty --always --long --match "*" 0.12.0dev3-33-g3bd37749 ``` --- versioneer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioneer.py b/versioneer.py index b4cd1d6c7c..83dfe215b4 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1165,7 +1165,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - TAG_PREFIX_REGEX = r"\*" + TAG_PREFIX_REGEX = r"*" # r"\*" - using escape on windows breaks tag extraction _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) From 78f6369bd4e558f719c83e7774f6d1f8e3a85f72 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 28 Feb 2022 13:37:45 -0600 Subject: [PATCH 229/229] test_service::test__version__ checks for untagged --- dpctl/tests/test_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index 291067b152..aadf320704 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -100,6 +100,7 @@ def test___version__(): dpctl_ver = getattr(dpctl, "__version__", None) assert type(dpctl_ver) is str assert "unknown" not in dpctl_ver + assert "untagged" not in dpctl_ver # Reg expr from PEP-440, relaxed to allow for semantic variant # 0.9.0dev0 allowed, vs. PEP-440 compliant 0.9.0.dev0 reg_expr = (