diff --git a/.flake8 b/.flake8 index 6e9c78b236..5a20d20b6e 100644 --- a/.flake8 +++ b/.flake8 @@ -13,6 +13,7 @@ exclude = .git per-file-ignores = + dpctl/_diagnostics.pyx: E999 dpctl/_sycl_context.pyx: E999, E225, E227 dpctl/_sycl_device.pyx: E999, E225 dpctl/_sycl_device_factory.pyx: E999, E225 @@ -23,6 +24,7 @@ per-file-ignores = dpctl/memory/_memory.pyx: E999, E225, E226, E227 dpctl/program/_program.pyx: E999, E225, E226, E227 dpctl/tensor/_usmarray.pyx: E999, E225, E226, E227 + dpctl/tensor/_dlpack.pyx: E999, E225, E226, E227 dpctl/tensor/numpy_usm_shared.py: F821 dpctl/tests/_cython_api.pyx: E999, E225, E227, E402 dpctl/utils/_compute_follows_data.pyx: E999, E225, E227 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..f10261fb12 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +dpctl/_version.py export-subst diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 4e72d54517..9e07dc8f69 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -1,6 +1,10 @@ name: Conda package -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: env: PACKAGE_NAME: dpctl @@ -115,6 +119,7 @@ jobs: - name: Add conda to system path run: echo $CONDA/bin >> $GITHUB_PATH - name: Install conda-build + # Needed to be able to run conda index run: conda install conda-build - name: Create conda channel run: | @@ -147,6 +152,11 @@ jobs: conda install $PACKAGE_NAME pytest python=${{ matrix.python }} $CHANNELS # Test installed packages conda list + - name: Smoke test + run: | + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + python -c "import dpctl; dpctl.lsplatform()" - name: Run tests run: | # echo "libintelocl.so" | tee /etc/OpenCL/vendors/intel-cpu.icd @@ -178,6 +188,7 @@ jobs: auto-activate-base: true activate-environment: "" - name: Install conda-build + # Needed to be able to run conda index run: conda install conda-build - name: Create conda channel run: | @@ -207,15 +218,52 @@ jobs: # Test installed packages conda list - name: Add library - run: echo "OCL_ICD_FILENAMES=C:\Miniconda\Library\lib\intelocl64.dll" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + run: | + echo "OCL_ICD_FILENAMES=C:\Miniconda\Library\lib\intelocl64.dll" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + try {$list = Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors | Select-Object -ExpandProperty Property } catch {$list=@()} + if ($list.count -eq 0) { + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos)) { + New-Item -Path HKLM:\SOFTWARE\Khronos + } + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos\OpenCL)) { + New-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL + } + if (-not (Test-Path -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors)) { + New-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors + } + New-ItemProperty -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors -Name C:\Miniconda\Library\lib\intelocl64.dll -Value 0 + try {$list = Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors | Select-Object -ExpandProperty Property } catch {$list=@()} + Write-Output $(Get-Item -Path HKLM:\SOFTWARE\Khronos\OpenCL\Vendors) + # Now copy OpenCL.dll into system folder + $system_ocl_icd_loader="C:\Windows\System32\OpenCL.dll" + $python_ocl_icd_loader="C:\Miniconda\Library\bin\OpenCL.dll" + Copy-Item -Path $python_ocl_icd_loader -Destination $system_ocl_icd_loader + if (Test-Path -Path $system_ocl_icd_loader) { + Write-Output "$system_ocl_icd_loader has been copied" + $acl = Get-Acl $system_ocl_icd_loader + Write-Output $acl + } else { + Write-Output "OCL-ICD-Loader was not copied" + } + # Variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default + echo "TBB_DLL_PATH=C:\Miniconda\Library\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + - name: Smoke test + run: | + set SYCL_ENABLE_HOST_DEVICE=1 + & { [Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Miniconda\Library\bin\", [EnvironmentVariableTarget]::Machine) } + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpctl; print(dpctl.get_devices(backend='opencl', device_type='gpu'))" + python -c "import dpctl; print(dpctl.get_num_devices(backend='opencl', device_type='gpu'))" - name: Run tests run: | set SYCL_ENABLE_HOST_DEVICE=1 + & { [Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Miniconda\Library\bin\", [EnvironmentVariableTarget]::Machine) } python -m pytest --pyargs ${{ env.MODULE_NAME }} upload_linux: needs: test_linux - if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true)}} + if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')}} runs-on: ubuntu-latest strategy: matrix: @@ -240,7 +288,7 @@ jobs: upload_windows: needs: test_windows - if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true)}} + if: ${{github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')}} runs-on: windows-latest strategy: matrix: @@ -263,3 +311,111 @@ jobs: run: | conda install anaconda-client anaconda --token ${{ env.ANACONDA_TOKEN }} upload --user dppy --label dev ${{ env.PACKAGE_NAME }}-*.tar.bz2 + + test_examples_linux: + needs: build_linux + runs-on: ${{ matrix.runner }} + strategy: + matrix: + python: [3.8] + experimental: [false] + runner: [ubuntu-latest] + continue-on-error: ${{ matrix.experimental }} + env: + CHANNELS: -c intel -c defaults --override-channels + + steps: + - name: Install conda-build + # Needed to be able to run conda index + run: conda install conda-build python=${{ matrix.python }} + - name: Checkout dpctl repo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python }} + - name: Add conda to system path + run: echo $CONDA/bin >> $GITHUB_PATH + - name: Create conda channel + run: | + mkdir -p $GITHUB_WORKSPACE/channel/linux-64 + mv ${PACKAGE_NAME}-*.tar.bz2 $GITHUB_WORKSPACE/channel/linux-64 + conda index $GITHUB_WORKSPACE/channel + # Test channel + conda search $PACKAGE_NAME -c $GITHUB_WORKSPACE/channel --override-channels + - name: Collect dependencies + run: | + CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" + conda install $PACKAGE_NAME python=${{ matrix.python }} $CHANNELS --only-deps --dry-run > lockfile + - name: Set pkgs_dirs + run: | + echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc + - name: Cache conda packages + uses: actions/cache@v2 + env: + CACHE_NUMBER: 0 # Increase to reset cache + with: + path: ~/.conda/pkgs + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-${{hashFiles('lockfile') }} + restore-keys: | + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}- + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- + - name: Install dpctl + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + conda activate + CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" + conda install -y $PACKAGE_NAME pytest python=${{ matrix.python }} $CHANNELS + # Test installed packages + conda list + - name: Install example requirements + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + conda install -y pybind11 cython + conda install -y -c intel mkl-dpcpp mkl-devel-dpcpp numba-dppy + conda create -y -n build_env -c intel dpcpp_linux-64 + - name: Build and run examples with native extensions + shell: bash -l {0} + run: | + source $CONDA/etc/profile.d/conda.sh + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + conda activate + cd examples/pybind11 + export CC=dpcpp + export CXX=dpcpp + for d in $(ls) + do + pushd $d + conda activate --stack build_env + python setup.py build_ext --inplace || exit 1 + conda deactivate + python example.py + popd + done + cd ../cython + for d in $(ls) + do + pushd $d + conda activate --stack build_env + python setup.py build_ext --inplace || exit 1 + conda deactivate + python run.py + popd + done + - name: Run Python examples + shell: bash -l {0} + run: | + cd examples/python + export OCL_ICD_FILENAMES=libintelocl.so + export SYCL_ENABLE_HOST_DEVICE=1 + for script in $(find . \( -not -name "_*" -and -name "*.py" \)) + do + echo "Executing ${script}" + python ${script} || exit 1 + done diff --git a/.github/workflows/cpp_style_checks.yml b/.github/workflows/cpp_style_checks.yml index b5a7def26f..b16530f967 100644 --- a/.github/workflows/cpp_style_checks.yml +++ b/.github/workflows/cpp_style_checks.yml @@ -16,7 +16,12 @@ jobs: steps: - uses: actions/checkout@v2 - name: Run clang-format style check for C/C++ programs. - uses: jidicula/clang-format-action@v3.1.0 + uses: jidicula/clang-format-action@v3.5.1 with: clang-format-version: '11' - check-path: 'dpctl-capi' + check-path: 'libsyclinterface' + - name: Run clang-format style check for api headers. + uses: jidicula/clang-format-action@v3.5.1 + with: + clang-format-version: '11' + check-path: 'dpctl/apis' diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index feb7e71015..c46af799a8 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -11,7 +11,7 @@ jobs: env: ONEAPI_ROOT: /opt/intel/oneapi - GTEST_ROOT: /home/runner/work/googletest-release-1.10.0/install + GTEST_ROOT: /home/runner/work/googletest-release-1.11.0/install steps: - name: Cancel Previous Runs @@ -29,17 +29,17 @@ jobs: - name: Install Intel OneAPI run: | - sudo apt-get install intel-oneapi-compiler-dpcpp-cpp=2021.3.0-3350 - sudo apt-get install intel-oneapi-tbb=2021.3.0-511 + sudo apt-get install intel-oneapi-compiler-dpcpp-cpp + sudo apt-get install intel-oneapi-tbb - - name: Install CMake + - name: Install CMake and Ninja run: | - sudo apt-get install cmake + sudo apt-get install cmake ninja-build - name: Setup Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' architecture: x64 - name: Cache Gtest @@ -47,8 +47,8 @@ jobs: uses: actions/cache@v2 with: path: | - /home/runner/work/googletest-release-1.10.0/install - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-release-1.10.0/install/include/gtest/*') }} + /home/runner/work/googletest-release-1.11.0/install + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-release-1.11.0/install/include/gtest/*') }} restore-keys: | ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- @@ -59,12 +59,12 @@ jobs: shell: bash -l {0} run: | cd /home/runner/work - wget https://github.com/google/googletest/archive/refs/tags/release-1.10.0.tar.gz - tar xf release-1.10.0.tar.gz - cd googletest-release-1.10.0 + wget https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz + tar xf release-1.11.0.tar.gz + cd googletest-release-1.11.0 mkdir build cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-release-1.10.0/install + cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-release-1.11.0/install make && make install - name: Checkout repo @@ -79,14 +79,29 @@ jobs: - name: Install dpctl dependencies shell: bash -l {0} run: | - pip install numpy cython setuptools pytest pytest-cov coverage[toml] + pip install numpy cython setuptools pytest pytest-cov scikit-build coverage[toml] - name: Build dpctl with coverage shell: bash -l {0} run: | source /opt/intel/oneapi/setvars.sh - python setup.py develop --coverage=True - python -c "import dpctl; print(dpctl.__version__); dpctl.lsplatform()" + export _SAVED_PATH=${PATH} + export PATH=$(dirname $(dirname $(which icx)))/bin-llvm:${PATH} + python setup.py develop -- \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER:PATH=icx \ + -DCMAKE_CXX_COMPILER:PATH=icpx \ + -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON \ + -DDPCTL_GENERATE_COVERAGE=ON \ + -DDPCTL_BUILD_CAPI_TESTS=ON \ + -DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=$(pwd) + pushd $(find _skbuild -name cmake-build) + cmake --build . --target lcov-genhtml || exit 1 + popd + export PATH=${_SAVED_PATH} + unset _SAVED_PATH + python -c "import dpctl; print(dpctl.__version__); dpctl.lsplatform()" || exit 1 pytest -q -ra --disable-warnings --cov-config pyproject.toml --cov dpctl --cov-report term-missing --pyargs dpctl -vv - name: Install coverall dependencies @@ -96,8 +111,9 @@ jobs: pip install coveralls - name: Upload coverage data to coveralls.io + shell: bash -l {0} run: | - coveralls-lcov -v -n build_cmake/tests/dpctl.lcov > dpctl-c-api-coverage.json + coveralls-lcov -v -n $(find _skbuild -name tests)/dpctl.lcov > dpctl-c-api-coverage.json coveralls --service=github --merge=dpctl-c-api-coverage.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index be23c32363..adf8717192 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -3,6 +3,8 @@ on: push: branches: - master + pull_request: + types: [opened, synchronize, reopened, closed] jobs: build-and-deploy: @@ -14,6 +16,7 @@ jobs: with: access_token: ${{ github.token }} - name: Add Intel repository + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB @@ -21,54 +24,61 @@ jobs: sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" sudo apt-get update - name: Install Intel OneAPI + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install intel-oneapi-dpcpp-cpp-compiler - name: Install Lua + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install liblua5.2-dev - name: Install Doxygen + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | sudo apt-get install doxygen - - name: Install CMake + - name: Install CMake and Ninja + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} run: | - sudo apt-get install cmake + sudo apt-get install cmake ninja-build - name: Setup Python + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} uses: actions/setup-python@v2 with: python-version: '3.8' architecture: x64 - name: Install sphinx dependencies + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | - pip install numpy cython setuptools sphinx sphinx_rtd_theme pydot graphviz + pip install numpy cython setuptools scikit-build sphinx sphinx_rtd_theme pydot graphviz sphinxcontrib-programoutput - name: Checkout repo uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Build dpctl - shell: bash -l {0} - run: | - source /opt/intel/oneapi/setvars.sh - python setup.py develop - python -c "import dpctl; print(dpctl.__version__)" - - name: Build docs + persist-credentials: false + - name: Build dpctl+docs + if: ${{ !github.event.pull_request || github.event.action != 'closed' }} shell: bash -l {0} run: | # Ensure that SYCL libraries are on LD_LIBRARY_PATH source /opt/intel/oneapi/setvars.sh - cd docs - mkdir -p build && cd build && rm -rf * wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-2.1.2/doxyrest-2.1.2-linux-amd64.tar.xz tar xf doxyrest-2.1.2-linux-amd64.tar.xz - cmake .. -DDPCTL_USE_MULTIVERSION_TEMPLATE=ON \ - -DDPCTL_ENABLE_DOXYREST=ON \ - -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 - make Sphinx - cd .. - mv generated_docs/docs ~/docs + python setup.py develop -- \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER:PATH=icx \ + -DCMAKE_CXX_COMPILER:PATH=icpx \ + -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON \ + -DDPCTL_GENERATE_DOCS=ON \ + -DDPCTL_ENABLE_DOXYREST=ON \ + -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 + python -c "import dpctl; print(dpctl.__version__)" || exit 1 + cd "$(find _skbuild -name cmake-build)" || exit 1 + cmake --build . --target Sphinx || exit 1 + mv ../cmake-install/docs/docs ~/docs git clean -dfx - cd .. - name: Publish docs + if: ${{ github.ref == 'refs/heads/master' }} shell: bash -l {0} run: | git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git @@ -77,9 +87,63 @@ jobs: echo `pwd` cd master git rm -rf * - mv ~/docs/* . + mv ~/docs/* . || exit 1 git add . git config --global user.name 'github-actions[doc-deploy-bot]' - git config --gloabl user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' git commit -m "Latest docs." git push tokened_docs gh-pages + - name: Publish pull-request docs + if: ${{ github.event.pull_request && github.event.action != 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + shell: bash -l {0} + run: | + git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git + git fetch tokened_docs + git checkout --track tokened_docs/gh-pages + echo `pwd` + [ -d pulls/${PR_NUM} ] && git rm -rf pulls/${PR_NUM} + mkdir -p pulls/${PR_NUM} + cd pulls/${PR_NUM} + mv ~/docs/* . + git add . + git config --global user.name 'github-actions[doc-deploy-bot]' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git commit -m "Docs for pull request ${PR_NUM}" + git push tokened_docs gh-pages + - name: Unpublished pull-request docs + if: ${{ github.event.pull_request && github.event.action == 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + shell: bash -l {0} + run: | + git remote add tokened_docs https://IntelPython:${{ secrets.GITHUB_TOKEN }}@github.com/IntelPython/dpctl.git + git fetch tokened_docs + git checkout --track tokened_docs/gh-pages + echo `pwd` + [ -d pulls/${PR_NUM} ] && git rm -rf pulls/${PR_NUM} + git config --global user.name 'github-actions[doc-deploy-bot]' + git config --global user.email 'github-actions[doc-deploy-bot]@users.noreply.github.com' + git commit -m "Removing docs for closed pull request ${PR_NUM}" + git push tokened_docs gh-pages + - name: Comment with URL to published pull-request docs + if: ${{ github.event.pull_request && github.event.action != 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + uses: mshick/add-pr-comment@v1 + with: + message: | + View rendered docs @ https://intelpython.github.io/dpctl/pulls/${{ env.PR_NUM }}/index.html + repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token-user-login: 'github-actions[bot]' + - name: Comment with URL about removal of PR docs + if: ${{ github.event.pull_request && github.event.action == 'closed' }} + env: + PR_NUM: ${{ github.event.number }} + uses: mshick/add-pr-comment@v1 + with: + message: | + Deleted rendered PR docs from intelpython.github.com/dpctl, latest should be updated shortly. :crossed_fingers: + repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token-user-login: 'github-actions[bot]' diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index b2101c24bb..c1c0329db7 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -10,9 +10,12 @@ jobs: runs-on: ubuntu-20.04 env: - OCLCPUEXP_FN: oclcpuexp-2021.12.6.0.19_rel.tar.gz - FPGAEMU_FN: fpgaemu-2021.12.6.0.19_rel.tar.gz - TBB_FN: oneapi-tbb-2021.4.0-lin.tgz + DOWNLOAD_URL_PREFIX: https://github.com/intel/llvm/releases/download + DRIVER_PATH: 2021-WW50 + OCLCPUEXP_FN: oclcpuexp-2021.13.11.0.23_rel.tar.gz + FPGAEMU_FN: fpgaemu-2021.13.11.0.23_rel.tar.gz + TBB_URL: https://github.com/oneapi-src/oneTBB/releases/download/v2021.5.0 + TBB_FN: oneapi-tbb-2021.5.0-lin.tgz steps: - name: Cancel Previous Runs @@ -45,19 +48,18 @@ jobs: if [[ -f bundle_id.txt && ( "$(cat bundle_id.txt)" == "${LATEST_LLVM_TAG_SHA}" ) ]]; then echo "Using cached download of ${LATEST_LLVM_TAG}" else - export DOWNLOAD_URL_PREFIX=https://github.com/intel/llvm/releases/download rm -rf dpcpp-compiler.tar.gz wget ${DOWNLOAD_URL_PREFIX}/${NIGHTLY_TAG}/dpcpp-compiler.tar.gz && echo ${LATEST_LLVM_TAG_SHA} > bundle_id.txt || rm -rf bundle_id.txt - [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-07/${OCLCPUEXP_FN} || rm -rf bundle_id.txt - [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/2021-07/${FPGAEMU_FN} || rm -rf bundle_id.txt - [ -f ${TBB_FN} ] || wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.4.0/${TBB_FN} || rm -rf bundle_id.txt + [ -f ${OCLCPUEXP_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/${DRIVER_PATH}/${OCLCPUEXP_FN} || rm -rf bundle_id.txt + [ -f ${FPGAEMU_FN} ] || wget ${DOWNLOAD_URL_PREFIX}/${DRIVER_PATH}/${FPGAEMU_FN} || rm -rf bundle_id.txt + [ -f ${TBB_FN} ] || wget ${TBB_URL}/${TBB_FN} || rm -rf bundle_id.txt rm -rf dpcpp_compiler tar xf dpcpp-compiler.tar.gz mkdir -p oclcpuexp mkdir -p fpgaemu [ -d oclcpuexp/x64 ] || tar xf ${OCLCPUEXP_FN} -C oclcpuexp [ -d fpgaemu/x64 ] || tar xf ${FPGAEMU_FN} -C fpgaemu - [ -d oneapi-tbb-2021.4.0/lib ] || tar xf ${TBB_FN} + [ -d oneapi-tbb-2021.5.0/lib ] || tar xf ${TBB_FN} mkdir -p dpcpp_compiler/lib mkdir -p dpcpp_compiler/lib/oclfpga touch dpcpp_compiler/lib/oclfpga/fpgavars.sh @@ -66,7 +68,7 @@ jobs: - name: Install system components shell: bash -l {0} run: | - sudo apt-get install cmake libtinfo5 + sudo apt-get install cmake ninja-build libtinfo5 - name: Setup Python uses: actions/setup-python@v2 @@ -77,7 +79,7 @@ jobs: - name: Install dpctl dependencies shell: bash -l {0} run: | - pip install numpy cython setuptools pytest + pip install numpy cython setuptools pytest scikit-build - name: Checkout repo uses: actions/checkout@v2 @@ -91,9 +93,10 @@ jobs: source ${SYCL_BUNDLE_FOLDER}/dpcpp_compiler/startup.sh export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oclcpuexp/x64:${LD_LIBRARY_PATH} export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/fpgaemu/x64:${LD_LIBRARY_PATH} - export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oneapi-tbb-2021.4.0/lib/intel64/gcc4.8:${LD_LIBRARY_PATH} + export LD_LIBRARY_PATH=${SYCL_BUNDLE_FOLDER}/oneapi-tbb-2021.5.0/lib/intel64/gcc4.8:${LD_LIBRARY_PATH} export OCL_ICD_FILENAMES=libintelocl.so:libintelocl_emu.so clang++ --version sycl-ls - python setup.py develop --sycl-compiler-prefix=$(dirname $(dirname `which clang++`)) - python -m pytest -v dpctl/tests + python setup.py develop -- -G Ninja -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -DDPCTL_DPCPP_HOME_DIR=$(dirname $(dirname $(which clang))) -DDPCTL_DPCPP_FROM_ONEAPI=OFF + python -c "import dpctl; dpctl.lsplatform()" || exit 1 + SYCL_ENABLE_HOST_DEVICE=1 python -m pytest -v dpctl/tests diff --git a/.github/workflows/python_style_checks.yml b/.github/workflows/python_style_checks.yml index 5e4b9a1d9f..92ae080f53 100644 --- a/.github/workflows/python_style_checks.yml +++ b/.github/workflows/python_style_checks.yml @@ -25,6 +25,10 @@ jobs: # The type of runner that the job will run on runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9] + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -33,16 +37,17 @@ jobs: - uses: actions/setup-python@v2 # Run black code formatter - - uses: psf/black@21.4b2 + - uses: psf/black@stable with: - args: ". --check" + src: "." + options: "--check" flake8: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.9] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 97cccfe031..ae22979595 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__/ # CMake build and local install directory build +_skbuild build_cmake install diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebc971590c..7311c9eb67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,17 +8,17 @@ repos: pass_filenames: false args: ["-r", "dpctl", "-lll"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.0.1 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 21.4b2 + rev: 22.1.0 hooks: - id: black exclude: "versioneer.py|dpctl/_version.py" - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort name: isort (python) @@ -29,7 +29,7 @@ repos: name: isort (pyi) types: [pyi] - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.1 + rev: 4.0.1 hooks: - id: flake8 - repo: https://github.com/pocc/pre-commit-hooks diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7a085d55..f14be39a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `dpctl.tensor.asarray`, `dpctl.tensor.empty` implemented (#646). +- `dpctl.tensor.usm_ndarray` adds support for DLPack protocol. `dpctl.tensor.from_dlpack` implemented (#682). + +### Changed +- dpctl-capi is now renamed to `libsyclinterface` (#666). + ## [0.11.4] - 12/03/2021 ### Fixed -* Fix tests for nested context factories expecting for integration environment by @PokhodenkoSA in https://github.com/IntelPython/dpctl/pull/705 +- Fix tests for nested context factories expecting for integration environment by @PokhodenkoSA in https://github.com/IntelPython/dpctl/pull/705 ## [0.11.3] - 11/30/2021 ### Fixed -* Set the last byte in allocated char array to zero [cherry picked from #650] (#699) +- Set the last byte in allocated char array to zero [cherry picked from #650] (#699) ## [0.11.2] - 11/29/2021 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..858c83c5fe --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.21...3.22 FATAL_ERROR) + +project(dpctl + LANGUAGES CXX + DESCRIPTION "Python interface for XPU programming" +) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(IntelDPCPP REQUIRED PATHS ${CMAKE_SOURCE_DIR}/cmake NO_DEFAULT_PATH) + +add_subdirectory(libsyclinterface) + +file(GLOB _dpctl_capi_headers dpctl/apis/include/*.h*) +install(FILES ${_dpctl_capi_headers} + DESTINATION dpctl/include +) + +add_subdirectory(dpctl) + +if (DPCTL_GENERATE_DOCS) + add_subdirectory(docs) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cae87e86e..f8d961169b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,12 +15,12 @@ Run before each commit: ```bash clang-format -style=file -i \ - dpctl-capi/include/*.h \ - dpctl-capi/include/Support/*.h \ - dpctl-capi/source/*.cpp \ - dpctl-capi/tests/*.cpp \ - dpctl-capi/helper/include/*.h \ - dpctl-capi/helper/source/*.cpp + libsyclinterface/include/*.h \ + libsyclinterface/include/Support/*.h \ + libsyclinterface/source/*.cpp \ + libsyclinterface/tests/*.cpp \ + libsyclinterface/helper/include/*.h \ + libsyclinterface/helper/source/*.cpp ``` > **_NOTE:_** A much simpler option is to use `pre-commit` and the @@ -161,13 +161,12 @@ these steps: 3. Build dpctl with code coverage support. ```bash - python setup.py develop --coverage=True - pytest -q -ra --disable-warnings --cov dpctl --cov-report term-missing --pyargs dpctl -vv + python scripts/gen_coverage.py --oneapi coverage html ``` Note that code coverage builds the C sources with debug symbols. For this - reason, the coverage flag is only available with the `develop` mode of + reason, the coverage script builds the package in `develop` mode of `setup.py`. The coverage results for the C and Python sources will be printed to the @@ -191,3 +190,52 @@ these steps: > ``` > The error is related to the `tcl` package. You should uninstall the `tcl` > package to resolve the error. + +## Error Reporting and Logging + +The SyclInterface library responds to `DPCTL_VERBOSITY` environment variable that controls the severity level of errors printed to console. +One can specify one of the following severity levels (in increasing order of severity): `warning` and `error`. + +```bash +export DPCTL_VERBOSITY=warning +``` + +Messages of a given severity are shown not only in the console for that severity, but also for the higher severity. For example, the severity level `warning` will output severity errors for `error` and `warning` to the console. + +### Optional use of the Google logging library (glog) + +Dpctl's error handler for libsyclinterface can be optionally configured to use [glog](https://github.com/google/glog). To use glog, follow the following steps: + +1. Install glog package of the latest version (0.5.0) + +```bash +conda install glog +``` +2. Build dpctl with glog support + +```bash +python scripts/build_locally.py --oneapi --glog +``` + +3. Use `dpctl._diagnostics.syclinterface_diagnostics(verbosity="warning", log_dir=None)` context manager to switch library diagnostics on for a block of Python code. +Use `DPCTLService_InitLogger` and `DPCTLService_ShutdownLogger` library C functions during library development to initialize the Google's logging library and de-initialize accordingly + +```python +from dpctl._diagnostics import syclinterface_diagnostics +import dpctl + +with syclinterface_diagnostics(): + code +``` + +```c +DPCTLService_InitLogger(const char *app_name, const char *log_dir); +DPCTLService_ShutdownLogger(); +``` + + - `*app_name` - name of the executable file (prefix for logs of various levels). + - `*log_dir` - directory path for writing log files. Specifying `NULL` results in logging to ``std::cerr``. + +> **_NOTE:_** +> +> If `InitGoogleLogging` is not called before first use of glog, the library will self-initialize to `logtostderr` mode and log files will not be generated. diff --git a/MANIFEST.in b/MANIFEST.in index 19f37e5a30..9ace2ea6ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ -include versioneer.py recursive-include dpctl/include *.h +include dpctl/include/dpctl4pybind11.hpp recursive-include dpctl *.pxd include dpctl/_sycl_context.h include dpctl/_sycl_context_api.h @@ -13,5 +13,6 @@ include dpctl/memory/_memory.h include dpctl/memory/_memory_api.h include dpctl/tensor/_usmarray.h include dpctl/tensor/_usmarray_api.h +recursive-include dpctl/tensor/include * include dpctl/tests/input_files/* include dpctl/tests/*.pyx diff --git a/README.md b/README.md index ea3e40d85f..031267f7e9 100644 --- a/README.md +++ b/README.md @@ -1,158 +1,114 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpctl/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpctl?branch=master) +![Generate Documentation](https://github.com/IntelPython/dpctl/actions/workflows/generate-docs.yml/badge.svg?branch=master) -About dpctl -=========== - -oneAPI logo - -`dpctl` (data parallel control) is a lightweight [Python package](https://intelpython.github.io/dpctl) exposing a -subset of the Intel(R) oneAPI DPC++ [runtime classes](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes) -that is distributed as part of [Intel(R) Distribution for Python*](https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/distribution-for-python.html) and -is included in Intel(R) [oneAPI](https://oneapi.io) [Base ToolKit](https://software.intel.com/content/www/us/en/develop/tools/oneapi/base-toolkit.html). -`dpctl` lets Python users query SYCL platforms, discover and represent SYCL devices, and construct SYCL queues to control data-parallel code execution on [Intel(R) XPUs](https://www.intel.com/content/www/us/en/newsroom/news/xpu-vision-oneapi-server-gpu.html) from Python. - -`dpctl` features classes representing [SYCL unified shared memory](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#sec:usm) -allocations as well as higher-level objects such as [`dpctl.tensor.usm_ndarray`](https://intelpython.github.io/dpctl/latest/docfiles/dpctl.tensor_api.html#module-dpctl.tensor) on top of USM allocations. - -`dpctl` assists authors of Python native extensions written in C, -Cython, or pybind11 to use its `dpctl.SyclQueue` object to indicate the offload -target as well as objects in `dpctl.memory` and `dpctl.tensor` submodules to -represent USM allocations that are accessible from within data-parallel code executed -on the target queue. - -`dpctl.tensor` submodule provides an array container representing an array in a -strided layout on top of a USM allocation. The submodule provides an array-API -conforming oneAPI DPC++ powered library to manipulate the array container. - -Requirements -============ -- Install Conda -- Install Intel oneAPI - - Set environment variable `ONEAPI_ROOT` - - Windows: `C:\Program Files (x86)\Intel\oneAPI\` - - Linux: `/opt/intel/oneapi` -- Install OpenCL HD graphics drivers - -Build and Install Conda Package -================================== -1. Create and activate conda build environment -```bash -conda create -n build-env conda-build -conda activate build-env -``` -2. Set environment variable `ONEAPI_ROOT` and build conda package -```bash -export ONEAPI_ROOT=/opt/intel/oneapi -conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel -``` -On Windows to cope with [long file names](https://github.com/IntelPython/dpctl/issues/15) -use `croot` with short folder path: -```cmd -set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" -conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" -``` - -:warning: **You could face issues with conda-build=3.20**: Use conda-build=3.18! +About +===== -3. Install conda package -```bash -conda install dpctl -``` +oneAPI logo + +Data Parallel Control (`dpctl`) is a Python library that allows a user +to *control* the execution placement of a [compute +kernel](https://en.wikipedia.org/wiki/Compute_kernel) on an +[XPU](https://www.intel.com/content/www/us/en/newsroom/news/xpu-vision-oneapi-server-gpu.html). +The compute kernel can be either a code written by the user, *e.g.*, +using `numba-dppy`, or a code that is part of a library like oneMKL. The `dpctl` +library is built upon the [SYCL +standard](https://www.khronos.org/sycl/) and implements Python +bindings for a subset of the standard [runtime +classes](https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes) +that allow users to query platforms, discover and represent devices +and sub-devices, and construct contexts and queues. In addition, +`dpctl` features classes for [SYCL Unified Shared Memory +(USM)](https://link.springer.com/chapter/10.1007/978-1-4842-5574-2_6) +management and implements a tensor [array +API](https://data-apis.org/array-api/latest/). + +The library also assists authors of Python native extensions written +in C, Cython, or pybind11 to access `dpctl` objects representing SYCL +devices, queues, memory, and tensors. + +`Dpctl` is the core part of a larger family of [data-parallel Python +libraries and +tools](https://www.intel.com/content/www/us/en/developer/tools/oneapi/distribution-for-python.html) +to program XPUs. The library is available via +[conda](https://anaconda.org/intel/dpctl) and +[pip](https://pypi.org/project/dpctl/). It is included in the [Intel(R) +Distribution for +Python*](https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/distribution-for-python.html) +(IDP). + +Installing +========== + +From Intel oneAPI +----------------- + +`dpctl` is packaged as part of the quarterly Intel oneAPI releases. To +get the library from the latest oneAPI release please follow the +instructions from Intel's [oneAPI installation +guide](https://www.intel.com/content/www/us/en/developer/articles/guide/installation-guide-for-oneapi-toolkits.html). +Note that you will need to install the Intel BaseKit toolkit to get +IDP and `dpctl`. + +From Conda +---------- + +`dpctl` package is available on the Intel channel on Annaconda +cloud. You an use the following to install `dpctl` from there: -Build and Install with setuptools -================================= -dpctl relies on DPC++ runtime. With Intel oneAPI installed you should activate it. -`setup.py` requires environment variable `ONEAPI_ROOT` and following packages -installed: -- `cython` -- `numpy` -- `cmake` - for building C API -- `ninja` - only on Windows - -You need DPC++ to build dpctl. If you want to build using the DPC++ in a -oneAPI distribution, activate DPC++ compiler as follows: ```bash -export ONEAPI_ROOT=/opt/intel/oneapi -source ${ONEAPI_ROOT}/compiler/latest/env/vars.sh +conda install dpctl -c intel ``` -For install: -```cmd -python setup.py install -``` - -For development: -```cmd -python setup.py develop -``` +From PyPi +--------- -It is also possible to build dpctl using [DPC++ toolchain](https://github.com/intel/llvm/blob/sycl/sycl/doc/GetStartedGuide.md) instead of oneAPI DPC++. Instead of activating the oneAPI environment, indicate the toolchain installation prefix with `--sycl-compiler-prefix` option, e.g. +`dpctl` is also available from PyPi and can be installed using: -```cmd -python setup.py develop --sycl-compiler-prefix=${DPCPP_ROOT}/llvm/build -``` - -Please use `python setup.py develop --help` for more details. - -Install Wheel Package from Pypi -================================== -1. Install dpctl -```cmd -python -m pip install --index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple dpctl +```bash +pip3 install dpctl ``` -Note: dpctl wheel package is placed on Pypi, but some of its dependencies (like Intel numpy) are in Anaconda Cloud. -That is why install command requires additional intel Pypi channel from Anaconda Cloud. -2. Set path to Performance Libraries in case of using venv or system Python: -On Linux: -```cmd -export LD_LIBRARY_PATH=/lib -``` -On Windows: -```cmd -set PATH=\bin;\Library\bin;%PATH% -``` +Installing the bleeding edge +------------------------ -Using dpctl -=========== -dpctl relies on DPC++ runtime. With Intel oneAPI installed you could activate it. +If you want to try out the current master, you can install it from our +development channel on Anaconda cloud: -On Windows: -```cmd -call "%ONEAPI_ROOT%\compiler\latest\env\vars.bat" -``` -On Linux: ```bash -source ${ONEAPI_ROOT}/compiler/latest/env/vars.sh +conda install dpctl -c dppy\label\dev ``` -When dpctl is installed via conda package -then it uses DPC++ runtime from `dpcpp_cpp_rt` package -and it is not necessary to activate oneAPI DPC++ compiler environment. +Building +======== -`dpcpp_cpp_rt` package is provided by oneAPI `conda_channel`. +Please refer our [getting started user +guide](https://intelpython.github.io/dpctl) for more information on +setting up a development environment and building `dpctl` from source. -Examples -======== +Running Examples +================ See examples in folder `examples`. Run python examples: + ```bash for script in `ls examples/python/`; do echo "executing ${script}"; python examples/python/${script}; done ``` Examples of building Cython extensions with DPC++ compiler, that interoperate -with dpctl can be found in folder `cython`. +with `dpctl` can be found in folder `cython`. Each example in `cython` folder can be built using `CC=icx CXX=dpcpp python setup.py build_ext --inplace`. Please refer to `run.py` script in respective folders to execute extensions. -Tests -===== -See tests in folder `dpctl/tests`. +Running Tests +============= +Tests are located in folder `dpctl/tests`. Run tests: ```bash diff --git a/cmake/IntelDPCPPConfig.cmake b/cmake/IntelDPCPPConfig.cmake new file mode 100644 index 0000000000..2b73830f1e --- /dev/null +++ b/cmake/IntelDPCPPConfig.cmake @@ -0,0 +1,293 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +IntelDPCPPConfig +------- + +DPCPP Library to verify DPCPP/SYCL compatability of CMAKE_CXX_COMPILER +and passes relevant compiler flags. + +Result Variables +^^^^^^^^^^^^^^^^ + +This will define the following variables: + +``IntelDPCPP_FOUND`` + True if the system has the DPCPP library. +``SYCL_LANGUAGE_VERSION`` + The SYCL language spec version by Compiler. +``SYCL_INCLUDE_DIR`` + Include directories needed to use SYCL. +``SYCL_IMPLEMENTATION_ID`` + The SYCL compiler variant. +``SYCL_FLAGS`` + SYCL specific flags for the compiler. + +Cache Variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``SYCL_INCLUDE_DIR`` + The directory containing ``sycl.hpp``. +``SYCL_LIBRARY_DIR`` + The path to the SYCL library. +``SYCL_FLAGS`` + SYCL specific flags for the compiler. +``SYCL_LANGUAGE_VERSION`` + The SYCL language spec version by Compiler. + + +.. note:: + + For now, user needs to set -DCMAKE_CXX_COMPILER or environment of + CXX pointing to SYCL compatible compiler ( eg: icx, clang++, icpx) + + Note: do not set to DPCPP compiler. If set to a Compiler family + that supports dpcpp ( eg: IntelLLVM) both DPCPP and SYCL + features are enabled. + + And add this package to user's Cmake config file. + + .. code-block:: cmake + + find_package(IntelDPCPP REQUIRED) + +#]=======================================================================] + +include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) + +find_package(PkgConfig QUIET) +if(PKG_CONFIG_FOUND) + # TODO add dependency package module checks, if any +endif() + + +# TODO: can't use find_program to override the CMAKE_CXX_COMPILER as +# Platform/ files are executed, potentially for a different compiler. +# Safer approach is to make user to define CMAKE_CXX_COMPILER. + +string(COMPARE EQUAL "${CMAKE_CXX_COMPILER}" "" nocmplr) +if(nocmplr) + set(IntelDPCPP_FOUND False) + set(SYCL_REASON_FAILURE "SYCL: CMAKE_CXX_COMPILER not set!!") + set(IntelDPCPP_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}") +endif() + +# Check for known compiler family that supports SYCL + +if( NOT "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xClang" AND + NOT "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xIntelLLVM") + set(IntelDPCPP_FOUND False) + set(SYCL_REASON_FAILURE "Unsupported compiler family ${CMAKE_CXX_COMPILER_ID} and compiler ${CMAKE_CXX_COMPILER}!!") + set(IntelDPCPP_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}") + return() +endif() + +# Assume that CXX Compiler supports SYCL and then test to verify. +set(SYCL_COMPILER ${CMAKE_CXX_COMPILER}) + + +# Function to write a test case to verify SYCL features. + +function(SYCL_FEATURE_TEST_WRITE src) + + set(pp_if "#if") + set(pp_endif "#endif") + + set(SYCL_TEST_CONTENT "") + string(APPEND SYCL_TEST_CONTENT "#include \nusing namespace std;\n") + string(APPEND SYCL_TEST_CONTENT "int main(){\n") + + # Feature tests goes here + + string(APPEND SYCL_TEST_CONTENT "${pp_if} defined(SYCL_LANGUAGE_VERSION)\n") + string(APPEND SYCL_TEST_CONTENT "cout << \"SYCL_LANGUAGE_VERSION=\"<=3.21 - python - - make # [unix] - - ninja # [win] + - ninja + - scikit-build - numpy 1.19 - wheel run: - python - {{ pin_compatible('numpy') }} - - dpcpp-cpp-rt >=2021.2 + - dpcpp-cpp-rt >=2022.0 test: requires: diff --git a/docs/.gitignore b/docs/.gitignore index c7b38c7ac8..c781c1805a 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,6 +1,7 @@ docs generated_docs -docfiles/dpctl-capi +docfiles/libsyclinterface +docfiles/dpctl api build conf.py diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index bba66d788a..ff3fbcb7cd 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -1,6 +1,3 @@ -cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -project("Data-parallel Control (dpctl)") - # Option to generate rst for C API and add to Sphinx documentation option(DPCTL_ENABLE_DOXYREST "Enable generation of rst files for C API" @@ -29,7 +26,7 @@ function(_setup_doxygen) if(DPCTL_ENABLE_DOXYGEN_HTML) set(GENERATE_HTML "YES") endif() - set(DOXYGEN_INPUT_DIR ../dpctl-capi/include) + set(DOXYGEN_INPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libsyclinterface/include) set(DOXYGEN_OUTPUT_DIR ${DOC_OUTPUT_DIR}/doxygen) set(DOXYGEN_INDEX_FILE ${DOXYGEN_OUTPUT_DIR}/xml/index.xml) set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) @@ -55,27 +52,29 @@ function(_setup_doxygen) # Target to generate only Doxygen documentation add_custom_target( Doxygen - ALL DEPENDS ${DOXYGEN_INDEX_FILE} ) endfunction() function(_setup_doxyrest) - set(DOXYREST_OUTPUT_DIR_NAME docfiles/dpctl-capi) + set(DOXYREST_OUTPUT_DIR_NAME docfiles/libsyclinterface) + # Set the DOXYREST_OUTPUT_DIR variable in both current and parent scope. + # The variable is used by _setup_sphinx when generating the conf.py file. set(DOXYREST_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DOXYREST_OUTPUT_DIR_NAME} PARENT_SCOPE ) set(DOXYREST_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DOXYREST_OUTPUT_DIR_NAME} - ) set(DOXYREST_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/doxyrest-config.lua.in) set(DOXYREST_CONFIG_OUT ${CMAKE_CURRENT_SOURCE_DIR}/doxyrest-config.lua) set(DOXYREST_OUTPUT ${DOXYREST_OUTPUT_DIR}/index.rst) set(DOXYGEN_OUTPUT_DIR ${DOC_OUTPUT_DIR}/doxygen) + configure_file(${DOXYREST_CONFIG_IN} ${DOXYREST_CONFIG_OUT} @ONLY) configure_file(${INDEX_DOXYREST_IN} ${INDEX_OUT} @ONLY) + add_custom_command( OUTPUT ${DOXYREST_OUTPUT} COMMAND @@ -83,73 +82,71 @@ function(_setup_doxyrest) ${DOXYREST_CONFIG_OUT} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS - # Other docs files that can be edited manually - ${INDEX_OUT} - ${DOXYGEN_INDEX_FILE} + # Other docs files that can be edited manually + ${INDEX_OUT} + ${DOXYGEN_INDEX_FILE} MAIN_DEPENDENCY ${DOXYREST_CONFIG_OUT} ${DOXYREST_CONFIG_IN} COMMENT "Generating Doxyrest documentation" ) # Target to generate rst from Doxygen XML using Doxyrest add_custom_target( Doxyrest - ALL DEPENDS Doxygen ${DOXYREST_OUTPUT} ) endfunction() function(_setup_sphinx) + set(GENERATE_RST_OUTPUT_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/docfiles/dpctl + ) set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}) set(SPHINX_OUTPUT_DIR ${DOC_OUTPUT_DIR}/docs) set(SPHINX_INDEX_FILE ${SPHINX_OUTPUT_DIR}/index.html) set(SPHINX_CONF_IN ${SPHINX_SOURCE}/conf.in) set(SPHINX_CONF_OUT ${SPHINX_SOURCE}/conf.py) - # Only regenerate Sphinx when: - # - Doxygen has rerun - # - Our doc files have been updated - # - The Sphinx config has been updated + set(DPCTL_PYAPI_RST_FILE ${GENERATE_RST_OUTPUT_DIR}/dpctl_pyapi.rst) + if(DPCTL_ENABLE_DOXYREST) - add_custom_command( - OUTPUT ${SPHINX_INDEX_FILE} - COMMAND - ${SPHINX_EXECUTABLE} -b html - ${SPHINX_SOURCE} - ${SPHINX_OUTPUT_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS - # Other docs files that can be edited manually - ${CMAKE_CURRENT_SOURCE_DIR}/index.rst - ${DOXYGEN_INDEX_FILE} - MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} - COMMENT "Generating Sphinx documentation" - ) - # Target to generate Sphinx - add_custom_target( - Sphinx - ALL - DEPENDS Doxyrest ${SPHINX_INDEX_FILE} - ) + set(DEPEND_ON_DOXYREST "Doxyrest") else() configure_file(${INDEX_NO_DOXYREST_IN} ${INDEX_OUT} @ONLY) - add_custom_command( - OUTPUT ${SPHINX_INDEX_FILE} - COMMAND - ${SPHINX_EXECUTABLE} -b html - ${SPHINX_SOURCE} - ${SPHINX_OUTPUT_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS - # Other docs files that can be edited manually - ${CMAKE_CURRENT_SOURCE_DIR}/index.rst - MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} - COMMENT "Generating Sphinx documentation" - ) - # Target to generate Sphinx - add_custom_target( - Sphinx - ALL - DEPENDS ${SPHINX_INDEX_FILE} - ) endif() + + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/generate_rst.py" + ${CMAKE_CURRENT_BINARY_DIR} + ) + # A custom command to generate the Python API rst files + add_custom_command( + OUTPUT ${DPCTL_PYAPI_RST_FILE} + COMMAND ${CMAKE_COMMAND} -E make_directory ${GENERATE_RST_OUTPUT_DIR} + COMMAND + "${Python_EXECUTABLE}" + "${CMAKE_CURRENT_BINARY_DIR}/generate_rst.py" + --dir "${GENERATE_RST_OUTPUT_DIR}" + --module "dpctl" + COMMENT "Generating RST files for Python API of dpctl" + ) + add_custom_command( + OUTPUT ${SPHINX_INDEX_FILE} + COMMAND + ${SPHINX_EXECUTABLE} -b html + ${SPHINX_SOURCE} + ${SPHINX_OUTPUT_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/index.rst + MAIN_DEPENDENCY ${SPHINX_CONF_OUT} ${SPHINX_CONF_IN} + COMMENT "Generating Sphinx documentation" + ) + # Target to generate Sphinx. Note that the order of the dependencies is + # important, we want the rst files to generate prior to sphinx build. + add_custom_target( + Sphinx + DEPENDS + ${DEPEND_ON_DOXYREST} + ${DPCTL_PYAPI_RST_FILE} + ${SPHINX_INDEX_FILE} + ) # Create a conf.py by replacing variables inside @@ with the current values configure_file(${SPHINX_CONF_IN} ${SPHINX_CONF_OUT} @ONLY) endfunction() @@ -185,18 +182,15 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) find_package(Git) find_package(Sphinx REQUIRED) find_package(Doxygen REQUIRED) +find_package(Python REQUIRED) + if (DPCTL_ENABLE_DOXYREST) find_package(Lua REQUIRED) find_package(Doxyrest REQUIRED) endif() # Set the location where the generated docs are saved -if(DPCTL_DOCGEN_PREFIX) - message(STATUS "Generating dpctl documents in " ${DPCTL_DOCGEN_PREFIX}) - set(DOC_OUTPUT_DIR ${DPCTL_DOCGEN_PREFIX}) -else() - set(DOC_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/generated_docs) -endif() +set(DOC_OUTPUT_DIR ${CMAKE_INSTALL_PREFIX}/docs) set(INDEX_NO_DOXYREST_IN ${CMAKE_CURRENT_SOURCE_DIR}/index_no_doxyrest.rst.in) set(INDEX_DOXYREST_IN ${CMAKE_CURRENT_SOURCE_DIR}/index_doxyrest.rst.in) @@ -208,3 +202,11 @@ if(DPCTL_ENABLE_DOXYREST) _setup_doxyrest() endif() _setup_sphinx() + +set_property( + DIRECTORY + PROPERTY + ADDITIONAL_CLEAN_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/docfiles/dpctl" + "${CMAKE_CURRENT_SOURCE_DIR}/docfiles/libsyclinterface" +) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index a1055b7256..49f8ec208b 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -562,7 +562,7 @@ INTERNAL_DOCS = NO # (including Cygwin) ands Mac users are advised to set this option to NO. # The default value is: system dependent. -CASE_SENSE_NAMES = YES +CASE_SENSE_NAMES = NO # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES, the @@ -917,11 +917,11 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = ../dpctl-capi/include/Support -EXCLUDE += ../dpctl-capi/include/Config -EXCLUDE += ../dpctl-capi/include/dpctl_vector.h -EXCLUDE += ../dpctl-capi/include/dpctl_data_types.h -EXCLUDE += ../dpctl-capi/include/dpctl_utils.h +EXCLUDE = ../libsyclinterface/include/Support +EXCLUDE += ../libsyclinterface/include/Config +EXCLUDE += ../libsyclinterface/include/dpctl_vector.h +EXCLUDE += ../libsyclinterface/include/dpctl_data_types.h +EXCLUDE += ../libsyclinterface/include/dpctl_utils.h # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/docs/README.md b/docs/README.md index 20e696f1a2..075889c9af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,58 +32,13 @@ sudo apt-get install liblua5.2-dev Generating the docs =================== -The documentation should be generated using the provided `Cmake` build script. -There are a few configurable options that can be used to select the type of -documentation to generate. +The helper script ``scripts/gen_docs.py`` is the preferred way to generate the +documentation. The generated documentation html pages will be installed to the +``CMAKE_INSTALL_PREFIX/docs`` directory. -Build only Doxygen for C API ---------------------------- ```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -make Doxygen +python scripts/gen_docs.py --doxyrest-root= ``` -The above steps will generate the `Doxygen` files at -`dpctl/docs/generated_docs/doxygen/html`. The documentation can also be -generated at a custom location by providing the optional flag - -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -DDPCTL_DOCGEN_PREFIX= -make Doxygen -``` - -Build only Sphinx for Python API --------------------------------- -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. -DDPCTL_DOCGEN_PREFIX= -make Sphinx -``` - -The `make Sphinx` command will generate only the Python API docs for dpctl. - -Build consolidated docs ------------------------ -It is possible to generate a single site with both Python and C API docs. As -mentioned before, `Doxyrest` and `Lua` are required to generate the consolidated -site. - -```bash -cd dpctl/docs -mkdir -p build -cd build -cmake .. \ - -DDPCTL_ENABLE_DOXYREST=ON \ - -DDoxyrest_DIR= \ - -DDPCTL_DOCGEN_PREFIX= -make Sphinx -``` -The `Doxyrest_DIR` flag is optional, but is needed when Doxyrest is installed in -a non-system location. +To skip generating the documentation for ``libsyclinterface``, the +``--doxyrest-root`` option should be omitted. diff --git a/docs/conf.in b/docs/conf.in index f904d1c26b..64af63037e 100644 --- a/docs/conf.in +++ b/docs/conf.in @@ -1,3 +1,19 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python3 # -*- coding: utf-8 -*- @@ -5,19 +21,23 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys + from docutils.parsers.rst import directives from sphinx.ext.autosummary import Autosummary, get_documenter from sphinx.util.inspect import safe_getattr import dpctl +sys.path.insert(0, os.path.abspath(".")) + +import extlinks_gen as urlgen + # -- Project information ----------------------------------------------------- project = "Data-parallel Control (dpctl)" -copyright = "2020, Intel Corp." +copyright = "2020-21, Intel Corp." author = "Intel Corp." version = dpctl.__version__.strip(".dirty") @@ -31,13 +51,15 @@ release = dpctl.__version__.strip(".dirty") # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx.ext.coverage", + "sphinx.ext.extlinks", + "sphinx.ext.githubpages", "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinxcontrib.programoutput", ] todo_include_todos = True @@ -168,7 +190,7 @@ class AutoAutoSummary(Autosummary): if not include_public: include_public = [] items = [] - for name in sorted(obj.__dict__.keys()): + for name in sorted(dir(obj)): try: documenter = get_documenter(app, safe_getattr(obj, name), obj) except AttributeError: @@ -209,3 +231,7 @@ class AutoAutoSummary(Autosummary): def setup(app): app.add_directive("autoautosummary", AutoAutoSummary) + + +# A dictionary of urls +extlinks = urlgen.create_extlinks() diff --git a/docs/docfiles/dpctl.memory_api.rst b/docs/docfiles/dpctl.memory_api.rst deleted file mode 100644 index c70041cdba..0000000000 --- a/docs/docfiles/dpctl.memory_api.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _dpctl.memory_api: - -############ -dpctl.memory -############ - -.. automodule:: dpctl.memory - -Classes -------- - -.. autoclass:: dpctl.memory.MemoryUSMDevice - :members: - :inherited-members: - :undoc-members: - -.. autoclass:: dpctl.memory.MemoryUSMHost - :members: - :inherited-members: - :undoc-members: - -.. autoclass:: dpctl.memory.MemoryUSMShared - :members: - :inherited-members: - :undoc-members: - -Functions ---------- - -.. autofunction:: dpctl.memory.as_usm_memory diff --git a/docs/docfiles/dpctl.program_api.rst b/docs/docfiles/dpctl.program_api.rst deleted file mode 100644 index 64db9c55e4..0000000000 --- a/docs/docfiles/dpctl.program_api.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _dpctl.program_api: - -############# -dpctl.program -############# - -.. automodule:: dpctl.program - -Classes -------- - -.. autoclass:: dpctl.program.SyclKernel - :members: - :undoc-members: - -.. autoclass:: dpctl.program.SyclProgram - :members: - :undoc-members: - -Exceptions ----------- - -.. autoexception:: dpctl.program.SyclProgramCompilationError - -Functions ---------- - -.. autofunction:: create_program_from_source -.. autofunction:: create_program_from_spirv diff --git a/docs/docfiles/dpctl.tensor_api.rst b/docs/docfiles/dpctl.tensor_api.rst deleted file mode 100644 index 837aa36ffb..0000000000 --- a/docs/docfiles/dpctl.tensor_api.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _dpctl.tensor_api: - -############ -dpctl.tensor -############ - -.. automodule:: dpctl.tensor - :members: - :undoc-members: diff --git a/docs/docfiles/dpctl_pyapi.rst b/docs/docfiles/dpctl_pyapi.rst deleted file mode 100644 index 7c60676d09..0000000000 --- a/docs/docfiles/dpctl_pyapi.rst +++ /dev/null @@ -1,86 +0,0 @@ -.. _dpctl_pyapi: - -################ -dpctl Python API -################ - -.. currentmodule:: dpctl - -.. automodule:: dpctl - -Sub-modules ------------ - - :mod:`dpctl.memory` - USM allocators and deallocators and classes that implement Python's - `buffer protocol`_. - :mod:`dpctl.program` - Experimental wrappers for SYCL 1.2 ``program`` and ``kernel`` classes. - The module is going to be refactored in the future to support SYCL - 2020's ``kernel_bundle`` feature and the wrapper for the ``program`` - class is going to be removed. - :mod:`dpctl.tensor` - Implementation of different types of tensor classes that use USM memory. - -Classes -------- - -.. toctree:: - :maxdepth: 1 - - dpctl.SyclContext : A Python class representing cl::sycl::context - dpctl.SyclDevice : A Python class representing cl::sycl::device - dpctl.SyclEvent : A Python class representing cl::sycl::event - dpctl.SyclPlatform : A Python class representing cl::sycl::event - dpctl.SyclQueue : A Python class representing cl::sycl::event - -Enumerations ------------- - -.. autoclass:: dpctl.backend_type - :members: - -.. autoclass:: dpctl.device_type - :members: - -Exceptions ----------- - -.. autoexception:: dpctl.SyclKernelInvalidRangeError -.. autoexception:: dpctl.SyclKernelSubmitError -.. autoexception:: dpctl.SyclQueueCreationError - -Device Selection Functions --------------------------- - -.. autofunction:: get_devices -.. autofunction:: select_accelerator_device -.. autofunction:: select_cpu_device -.. autofunction:: select_default_device -.. autofunction:: select_gpu_device -.. autofunction:: select_host_device -.. autofunction:: get_num_devices -.. autofunction:: has_cpu_devices -.. autofunction:: has_gpu_devices -.. autofunction:: has_accelerator_devices -.. autofunction:: has_host_device - -Queue Management Functions --------------------------- - -.. autofunction:: device_context -.. autofunction:: get_current_backend -.. autofunction:: get_current_device_type -.. autofunction:: get_current_queue -.. autofunction:: get_num_activated_queues -.. autofunction:: is_in_device_context -.. autofunction:: set_global_queue - -Other Helper Functions ----------------------- -.. autofunction:: get_platforms -.. autofunction:: lsplatform - -.. _Section 4.6: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes -.. _SYCL 2020 spec: https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html -.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html diff --git a/docs/docfiles/dpctl_pyapi/SyclContext.rst b/docs/docfiles/dpctl_pyapi/SyclContext.rst deleted file mode 100644 index 8ec8728d04..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclContext.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. _SyclContext_api: - -################# -dpctl.SyclContext -################# - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclContext - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclContext - :attributes: - - .. rubric:: Private methods: - - .. autoautosummary:: dpctl.SyclContext - :private_methods: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclContext - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclContext.device_count - - -Private methods ---------------- - -.. autofunction:: dpctl.SyclContext._get_capsule - - -Public methods --------------- - -.. autofunction:: dpctl.SyclContext.addressof_ref -.. autofunction:: dpctl.SyclContext.get_devices diff --git a/docs/docfiles/dpctl_pyapi/SyclDevice.rst b/docs/docfiles/dpctl_pyapi/SyclDevice.rst deleted file mode 100644 index 43a13ce2fe..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclDevice.rst +++ /dev/null @@ -1,84 +0,0 @@ -.. _SyclDevice_api: - -################ -dpctl.SyclDevice -################ - -.. currentmodule:: dpctl - -.. autoclass:: SyclDevice - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclDevice - :attributes: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclDevice - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclDevice.backend -.. autoattribute:: dpctl.SyclDevice.default_selector_score -.. autoattribute:: dpctl.SyclDevice.device_type -.. autoattribute:: dpctl.SyclDevice.driver_version -.. autoattribute:: dpctl.SyclDevice.filter_string -.. autoattribute:: dpctl.SyclDevice.has_aspect_accelerator -.. autoattribute:: dpctl.SyclDevice.has_aspect_cpu -.. autoattribute:: dpctl.SyclDevice.has_aspect_custom -.. autoattribute:: dpctl.SyclDevice.has_aspect_fp16 -.. autoattribute:: dpctl.SyclDevice.has_aspect_fp64 -.. autoattribute:: dpctl.SyclDevice.has_aspect_gpu -.. autoattribute:: dpctl.SyclDevice.has_aspect_host -.. autoattribute:: dpctl.SyclDevice.has_aspect_image -.. autoattribute:: dpctl.SyclDevice.has_aspect_int64_base_atomics -.. autoattribute:: dpctl.SyclDevice.has_aspect_int64_extended_atomics -.. autoattribute:: dpctl.SyclDevice.has_aspect_online_compiler -.. autoattribute:: dpctl.SyclDevice.has_aspect_online_linker -.. autoattribute:: dpctl.SyclDevice.has_aspect_queue_profiling -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_device_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_host_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_restricted_shared_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_shared_allocations -.. autoattribute:: dpctl.SyclDevice.has_aspect_usm_system_allocator -.. autoattribute:: dpctl.SyclDevice.image_2d_max_height -.. autoattribute:: dpctl.SyclDevice.image_2d_max_width -.. autoattribute:: dpctl.SyclDevice.image_3d_max_depth -.. autoattribute:: dpctl.SyclDevice.image_3d_max_height -.. autoattribute:: dpctl.SyclDevice.image_3d_max_width -.. autoattribute:: dpctl.SyclDevice.is_accelerator -.. autoattribute:: dpctl.SyclDevice.is_cpu -.. autoattribute:: dpctl.SyclDevice.is_gpu -.. autoattribute:: dpctl.SyclDevice.is_host -.. autoattribute:: dpctl.SyclDevice.max_compute_units -.. autoattribute:: dpctl.SyclDevice.max_num_sub_groups -.. autoattribute:: dpctl.SyclDevice.max_read_image_args -.. autoattribute:: dpctl.SyclDevice.max_work_group_size -.. autoattribute:: dpctl.SyclDevice.max_work_item_dims -.. autoattribute:: dpctl.SyclDevice.max_work_item_sizes -.. autoattribute:: dpctl.SyclDevice.max_write_image_args -.. autoattribute:: dpctl.SyclDevice.name -.. autoattribute:: dpctl.SyclDevice.parent_device -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_char -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_double -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_float -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_half -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_int -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_long -.. autoattribute:: dpctl.SyclDevice.preferred_vector_width_short -.. autoattribute:: dpctl.SyclDevice.sub_group_independent_forward_progress -.. autoattribute:: dpctl.SyclDevice.vendor - -Public methods --------------- - -.. autofunction:: dpctl.SyclDevice.addressof_ref -.. autofunction:: dpctl.SyclDevice.create_sub_devices -.. autofunction:: dpctl.SyclDevice.get_filter_string -.. autofunction:: dpctl.SyclDevice.print_device_info diff --git a/docs/docfiles/dpctl_pyapi/SyclEvent.rst b/docs/docfiles/dpctl_pyapi/SyclEvent.rst deleted file mode 100644 index bf6dbdef62..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclEvent.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _SyclEvent_api: - -############### -dpctl.SyclEvent -############### - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclEvent - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclEvent - :methods: - -Detail -====== - -.. autofunction:: dpctl.SyclEvent.addressof_ref -.. autofunction:: dpctl.SyclEvent.wait diff --git a/docs/docfiles/dpctl_pyapi/SyclPlatform.rst b/docs/docfiles/dpctl_pyapi/SyclPlatform.rst deleted file mode 100644 index fa5dafe93d..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclPlatform.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. _SyclPlatform_api: - -################## -dpctl.SyclPlatform -################## - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclPlatform - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclPlatform - :attributes: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclPlatform - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclPlatform.backend -.. autoattribute:: dpctl.SyclPlatform.name -.. autoattribute:: dpctl.SyclPlatform.vendor -.. autoattribute:: dpctl.SyclPlatform.version - -Public methods --------------- - -.. autofunction:: dpctl.SyclPlatform.print_platform_info diff --git a/docs/docfiles/dpctl_pyapi/SyclQueue.rst b/docs/docfiles/dpctl_pyapi/SyclQueue.rst deleted file mode 100644 index a48018d6e1..0000000000 --- a/docs/docfiles/dpctl_pyapi/SyclQueue.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _SyclQueue_api: - -############### -dpctl.SyclQueue -############### - -.. currentmodule:: dpctl - -.. autoclass:: dpctl.SyclQueue - - .. rubric:: Attributes: - - .. autoautosummary:: dpctl.SyclQueue - :attributes: - - .. rubric:: Private methods: - - .. autoautosummary:: dpctl.SyclQueue - :private_methods: - - .. rubric:: Public methods: - - .. autoautosummary:: dpctl.SyclQueue - :methods: - -Detail -====== - -Attributes ----------- - -.. autoattribute:: dpctl.SyclQueue.is_in_order -.. autoattribute:: dpctl.SyclQueue.sycl_context -.. autoattribute:: dpctl.SyclQueue.sycl_device - -Private methods ---------------- - -.. autofunction:: dpctl.SyclQueue._get_capsule - - -Public methods --------------- - -.. autofunction:: dpctl.SyclQueue.addressof_ref -.. autofunction:: dpctl.SyclQueue.get_sycl_backend -.. autofunction:: dpctl.SyclQueue.get_sycl_context -.. autofunction:: dpctl.SyclQueue.get_sycl_device -.. autofunction:: dpctl.SyclQueue.mem_advise -.. autofunction:: dpctl.SyclQueue.memcpy -.. autofunction:: dpctl.SyclQueue.prefetch -.. autofunction:: dpctl.SyclQueue.submit -.. autofunction:: dpctl.SyclQueue.wait diff --git a/docs/docfiles/intro.rst b/docs/docfiles/intro.rst index 115749b3da..892e66af72 100644 --- a/docs/docfiles/intro.rst +++ b/docs/docfiles/intro.rst @@ -2,10 +2,9 @@ Welcome to Data-parallel Control (dpctl)'s documentation! ========================================================= The data-parallel control (dpctl) library provides C and Python bindings for -`SYCL 2020 `_. -The SYCL 2020 features supported by dpctl are limited to those included by -Intel's DPCPP compiler and specifically cover the SYCL runtime classes described -in `Section 4.6 `_ +:sycl_spec_2020:`SYCL 2020 <>`. The SYCL 2020 features supported by dpctl are +limited to those included by Intel's DPCPP compiler and specifically cover the +SYCL runtime classes described in :sycl_runtime_classes:`Section 4.6 <>` of the SYCL 2020 specification. Apart from the bindings for these runtime classes, dpctl includes bindings for SYCL USM memory allocators and deallocators. Dpctl's Python API provides classes that implement diff --git a/docs/docfiles/urls.json b/docs/docfiles/urls.json new file mode 100644 index 0000000000..3e0906fc41 --- /dev/null +++ b/docs/docfiles/urls.json @@ -0,0 +1,16 @@ +{ + "dpcpp_envar": "https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md", + "numa_domain": "https://en.wikipedia.org/wiki/Non-uniform_memory_access", + "oneapi": "https://www.oneapi.io/", + "oneapi_filter_selection": "https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/FilterSelector/FilterSelector.adoc", + "sycl_aspects": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#table.device.aspect", + "sycl_context": "https://sycl.readthedocs.io/en/latest/iface/context.html", + "sycl_device": "https://sycl.readthedocs.io/en/latest/iface/device.html", + "sycl_device_info": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_device_information_descriptors", + "sycl_device_selector": "https://sycl.readthedocs.io/en/latest/iface/device-selector.html", + "sycl_event": "https://sycl.readthedocs.io/en/latest/iface/event.html", + "sycl_platform": "https://sycl.readthedocs.io/en/latest/iface/platform.html", + "sycl_queue": "https://sycl.readthedocs.io/en/latest/iface/queue.html", + "sycl_runtime_classes": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html#_sycl_runtime_classes", + "sycl_spec_2020": "https://www.khronos.org/registry/SYCL/specs/sycl-2020/html/sycl-2020.html" +} diff --git a/docs/docfiles/user_guides/QuickStart.rst b/docs/docfiles/user_guides/QuickStart.rst index 7b377a94b6..94b4953ca3 100644 --- a/docs/docfiles/user_guides/QuickStart.rst +++ b/docs/docfiles/user_guides/QuickStart.rst @@ -4,14 +4,8 @@ Quick Start Guide ################# - -.. contents:: Table of contents - :local: - :backlinks: none - :depth: 3 - Installing from oneAPI ----------------------- +====================== Dpctl is available as part of the oneAPI Intel Distribution of Python (IDP). Please follow `oneAPI installation guide`_ to install oneAPI. In this quick @@ -50,13 +44,13 @@ On Windows `GPU driver installation guide`_. Install Wheel package from Pypi -------------------------------- +=============================== Dpctl can also be istalled from Pypi. .. code-block:: bash - python -m pip install --index-url https://pypi.anaconda.org/intel/simple -extra-index-url https://pypi.org/simple dpctl + python -m pip install --index-url https://pypi.anaconda.org/intel/simple dpctl .. note:: @@ -79,7 +73,7 @@ On Windows set PATH=\bin;\Library\bin;%PATH% Building from source --------------------- +==================== To build dpctl from source, we need dpcpp and GPU drivers (and optionally CPU OpenCL drivers). It is preferable to use the dpcpp compiler packaged as part of @@ -87,12 +81,13 @@ oneAPI. However, it is possible to use a custom build of dpcpp to build dpctl, especially if you want to enable CUDA support. Building using oneAPI dpcpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- As before, oneAPI and graphics drivers should be installed on the system prior to proceeding further. -**Activate oneAPI as follows** +Activate oneAPI as follows +~~~~~~~~~~~~~~~~~~~~~~~~~~ On Linux @@ -106,7 +101,8 @@ On Windows call "%ONEAPI_ROOT%\setvars.bat" -**Build and install using conda-build** +Build and install using conda-build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The conda-recipe included with the sources can be used to build the dpctl package. The advantage of this approach is that all dependencies are pulled in @@ -136,52 +132,67 @@ After building the conda package you may install it by executing: You could face issues with conda-build version 3.20. Use conda-build 3.18 instead. -**Build and Install with setuptools** -To build using Python ``setuptools``, the following packages should be +Build and install with scikit-build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To build using Python ``setuptools`` and ``scikit-build``, the following Python packages should be installed: - ``cython`` - ``numpy`` - ``cmake`` - - ``ninja`` (only on Windows) + - ``scikit-build`` + - ``ninja`` - ``gtest`` (optional to run C API tests) + - ``gmock`` (optional to run C API tests) - ``pytest`` (optional to run Python API tests) -Once the prerequisites are installed, building using ``setuptools`` involves The -usual steps +Once the prerequisites are installed, building using ``scikit-build`` involves the usual steps, to build and install: + +.. code-block:: bash + + python setup.py install -- -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -to build and install +, and to develop: .. code-block:: bash - python setup.py install + python setup.py develop -G Ninja -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ON -, and to develop. +On Windows, use ``icx`` for both C and CXX compilers. + +Developing on Linux can also be done using driver script: .. code-block:: bash - python setup.py develop + python scripts/build_locally.py --oneapi + Building using custom dpcpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- It is possible to build dpctl from source using .. _DPC++ toolchain: https://github.com/intel/llvm/blob/sycl/sycl/doc/GetStartedGuide.md instead of the DPC++ compiler that comes with oneAPI. One reason for doing this may be to enable support for CUDA devices. -Following steps in :ref:`Build and Install with setuptools` use command line -option :code:`--sycl-compiler-prefix`, for example: +Following steps in `Build and install with scikit-build`_ use command line option to set relevant cmake variables, for example: .. code-block:: bash - python setup.py develop --sycl-compiler-prefix=${DPCPP_ROOT}/llvm/build + python setup.py develop -- -G Ninja -DCMAKE_C_COMPILER:PATH=clang -DCMAKE_CXX_COMPILER:PATH=clang++ -DDPCTL_ENABLE_LO_PROGRAM_CREATION=ONE -DDPCTL_DPCPP_HOME_DIR=${DPCPP_ROOT}/llvm/build -DDPCTL_DPCPP_FROM_ONEAPI=OFF + +Alterantively, the driver script can be used + +.. code-block:: bash + + python scripts/build_locally.py --c-compiler=clang --cxx-compiler=clang++ --compiler-root=${DPCPP_ROOT}/llvm/build Available options and their descriptions can be retrieved using option :code:`--help`. Using dpctl ------------ +=========== Dpctl requires a DPC++ runtime. When dpctl is installed via conda then it uses the DPC++ runtime from ``dpcpp_cpp_rt`` package that is part of IDP. When using @@ -190,10 +201,10 @@ the system. The easiest way to setup a DPC++ runtime will be by activating oneAPI. Running examples and tests --------------------------- +========================== Running the examples -~~~~~~~~~~~~~~~~~~~~ +-------------------- After setting up dpctl you can try out the Python examples as follows: @@ -213,7 +224,7 @@ located under *examples/cython*. Each example in the folder can be built using examples. Running the Python tests -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The dpctl Python test suite can be executed as follows: @@ -222,14 +233,13 @@ The dpctl Python test suite can be executed as follows: pytest --pyargs dpctl -Building the C API shared library ---------------------------------- +Building the DPCTLSyclInterface library +======================================= -The dpctl C API is a shared library called libDPCTLSyclInterface and is built -together when build the Python package. However, it is possible to only build -the C API as a standalone library. To do so, you will need ``cmake``, +The libDPCTLSyclInterface is a shared library used by the Python package. +To build the library you will need ``DPC++`` toolchain, ``cmake``, ``ninja`` or ``make``, and optionally ``gtest 1.10`` if you wish to run the -C API test suite. +test suite. For example, on Linux the following script can be used to build the C oneAPI library. diff --git a/docs/docfiles/user_guides/UserManual.rst b/docs/docfiles/user_guides/UserManual.rst new file mode 100644 index 0000000000..9b955f1b0c --- /dev/null +++ b/docs/docfiles/user_guides/UserManual.rst @@ -0,0 +1,10 @@ +.. _user_manual: + +########### +User Manual +########### + +.. toctree:: + :maxdepth: 3 + + manual/dpctl/intro diff --git a/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst b/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst new file mode 100644 index 0000000000..fbda045899 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/basic_concepts.rst @@ -0,0 +1,75 @@ +.. _basic_concepts: + +Basic Concepts +============== + +The section introduces the basic concepts for XPU management used by dpctl. +As dpctl is based on SYCL the concepts should be familiar to users with prior +experience with SYCL. However, users of dpctl need not have any prior experience +with SYCL and the concepts presented here should be self-sufficient. We do not +go into all the SYCL-level details here and if needed readers should refer to a +more topical SYCL reference such as the :sycl_spec_2020:`SYCL 2020 spec <>`. + +* **Heterogeneous computing** + Refers to using multiple devices in a program. + +* **Host** + Every program starts by running on a host, and most of the lines of code in + a program, in particular lines of code implementing the Python interpreter + itself, are usually for the host. Hosts are customarily CPUs. + +* **Device** + A device is an XPU connected to a host that is programmable with a specific + device driver. Different types of devices can have different architectures + (CPUs, GPUs, FPGA, ASICs, DSP), but are programmable using the same + :oneapi:`oneAPI <>` programming model. + +* **Platform** + A device driver installed on the system is termed as a platform. As multiple + devices of the same type can share the same device driver, a platform may + contain multiple devices. Note that the same physical hardware (say, a GPU) + may be reflected as two separate devices if they can be programmed by more + than one platform, *e.g.*, the same GPU hardware can be listed as an + OpenCL GPU device and a Level-Zero GPU device. + +* **Context** + A context holds the run-time information needed to operate on a device or a + group of devices from the same platform. Contexts are relatively expensive + to create and should be reused as much as possible. + +* **Queue** + A queue is needed to schedule execution of any computation, or data + copying on the device. Queue construction requires specifying a device + and a context targeting that device as well as additional properties, + such as whether profiling information should be collected or whether submitted + tasks are executed in the order in which they were submitted. + +* **Event** + An event holds information related to computation/data movement operation + scheduled for execution on a queue, such as its execution status as well + as profiling information if the queue the task was submitted to allowed + for collection of such information. Events can be used to specify task + dependencies as well as to synchronize host and devices. + +* **USM** + Unified Shared Memory (USM) refers to pointer based device memory management. + USM allocations are bound to context. In other words, a pointer representing + USM allocation can be unambiguously mapped to the data it represents only + if the associated context is known. USM allocations are accessible by + computational kernels that are executed on a device, provided that the + allocation is bound to the same context that was used to construct the queue + where the kernel was scheduled for execution. + + Depending on the capability of the device, USM allocations can be a "device" + allocation, a "shared" allocation, or a "host" allocation. A "device" + allocation is not accessible from host, while "shared" or "host" allocations + are. "Host" allocation refers to an allocation in host memory that is + accessible from a device. + + "Shared" allocations are accessible by both host and device. Runtime manages + synchronization of host's and device's view into shared allocations. Initial + placement of the shared allocations is not defined. + +* **Backend** + Refers to an implementation of :oneapi:`oneAPI <>` programming model exposed + by the underlying runtime. diff --git a/docs/docfiles/user_guides/manual/dpctl/device_selection.rst b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst new file mode 100644 index 0000000000..b68f930834 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/device_selection.rst @@ -0,0 +1,152 @@ +.. _device_selection: + +################ +Device Selection +################ + +Device selection refers to programmatically selecting a single device from +the set of :ref:`devices ` available on the system. + +Selecting a Specific Type of Device +----------------------------------- + +If a user needs to select a specific type of device such as a GPU, they can +directly use one of the helper functions included inside dpctl. Dpctl includes +:ref:`helper functions ` for selecting +a ``host``, a ``cpu``, a ``gpu``, an ``accelerator``, or the ``default`` device. +These functions are analogous to SYCL's built-in +:sycl_device_selector:`sycl::device_selector <>` classes. The scoring and +selection of a specific device when multiple devices of the same type are +available on a system is deferred to the underlying SYCL runtime. + +The example :ref:`fig-gpu-device-selection` shows the usage of the +:func:`dpctl.select_gpu_device()` device selection function. In case multiple +GPU devices are available, only one is returned based on the underlying scoring +logic inside the SYCL runtime. If the selection function was unable to select a +device a ``ValueError`` is raised. + +.. _fig-gpu-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 38-52 + :caption: Selecting a GPU Device + :linenos: + +A possible output for the example :ref:`fig-gpu-device-selection` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_gpu_device + +.. _sec-filter-selection: + +Selecting a Device Using a Filter String +---------------------------------------- + +Along with using the default device selection functions, a more explicit way of +device selection involves the use of *filter strings* (refer +:oneapi_filter_selection:`oneAPI filter selection extension <>`). The example +:ref:`fig-gpu-device-selection` also demonstrates the use of a filter string +to create a GPU device directly. Using a filter string allows much more +fine-grained control for selecting a device. The following example +:ref:`fig-filter-selection` demonstrates usages of device selection using filter +strings. + +.. _fig-filter-selection: + +.. literalinclude:: ../../../../../examples/python/filter_selection.py + :language: python + :lines: 20-21, 23-53 + :caption: Device Creation With Filter Strings + :linenos: + +A possible output for the example :ref:`fig-filter-selection` may be: + +.. program-output:: python ../examples/python/filter_selection.py -r select_using_filter + + +It is also possible to pass a list of devices using a filter string. The +example :ref:`fig-adv-device-selection` demonstrates such a use case. The +filter string ``gpu,cpu`` implies that a GPU should be selected if available, +else a CPU device should be selected. + +.. _fig-adv-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 55-67 + :caption: Selecting a GPU Device if Available + :linenos: + +A possible output for the example :ref:`fig-adv-device-selection` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_gpu_device_if_present + +.. Note:: + A **filter string** is a three-tuple that may specify the *backend*, + *device type*, and *device number* as a colon (:) separated string. The + backend specifies the type of device driver and can have a value such as + *host*, *opencl*, *level-zero*, or *cuda*. The device type can be *host*, + *gpu*, *cpu*, *accelerator*. And, the device number is a numeric value + specifying the ordinality of the device in the listing of devices as + determined by the SYCL runtime. Each of the backend, device type, and device + number value is optional, but at least one of them should be provided, + *i.e.*, ``opencl:gpu:0``, ``gpu:0``, ``gpu``, ``0``, and ``opencl:0`` are + all valid filter strings. + + The device listing including the device number value remain stable for + a given system unless the driver configuration is changed or the SYCL + runtime setting is changed using the ``SYCL_DEVICE_FILTER`` environment + variable. Please refer + :oneapi_filter_selection:`oneAPI filter selection extension <>` for more + detail. + +Advanced Device Selection +------------------------- + +Till now we have discussed device selection using methods that defer the +selection logic to the SYCL runtime. However, real-world applications may +require more precise control over device selection. Dpctl offers a way for users +to accomplish more advanced device selection. + +.. _fig-custom-device-selection: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 70-91 + :caption: Custom Device Selection + :linenos: + +The example :ref:`fig-custom-device-selection` shows a way of selecting a device +based off a specific hardware property. The :func:`dpctl.get_devices()` returns +a list of all *root* devices on the system, out of that list the devices that +support half-precision floating-point arithmetic are selected. Finally, a +"score" computed using the SYCL runtime's default device scoring logic that is +stored in :attr:`dpctl.SyclDevice.default_selector_score` is used to select a +single device. Refer the documentation of :class:`dpctl.SyclDevice` for a list +of hardware properties that may be used for device selection. + +.. _RootDevice: + +.. Note:: + A **root** device implies an unpartitioned device. A root device can be + partitioned into two or more :ref:`sub-devices ` + based on various criteria. For example, a CPU device with multiple NUMA + domains may be partitioned into multiple sub-devices, each representing a + sub-device. + +A convenience function :func:`dpctl.select_device_with_aspects()` is available +that makes it easy to select a device based on a set of specific aspects. The +example :ref:`fig-select-device-with-aspects` selects a device that +supports double precision arithmetic and SYCL USM shared memory allocation. + +.. _fig-select-device-with-aspects: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-21, 94-103 + :caption: Device Selection Using Aspects + :linenos: + +A possible output for the example :ref:`fig-select-device-with-aspects` may be: + +.. program-output:: python ../examples/python/device_selection.py -r create_device_with_aspects diff --git a/docs/docfiles/user_guides/manual/dpctl/devices.rst b/docs/docfiles/user_guides/manual/dpctl/devices.rst new file mode 100644 index 0000000000..ca2529b0f4 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/devices.rst @@ -0,0 +1,141 @@ +.. _devices: + +###### +Device +###### + +A device is an abstract representation for an XPU. The :class:`dpctl.SyclDevice` +class represents a device and is a wrapper over the +:sycl_device:`sycl::device <>` SYCL runtime class. + +Creating Devices +---------------- + +We touched upon device creation under the :ref:`device_selection` section. the +:class:`dpctl.SyclDevice` class includes a default constructor to create a +"default" device that is selected by the SYCL runtime. Users can also use +explicit :ref:`filter selector strings ` to create a +device. + +Listing Devices +--------------- + +:py:mod:`dpctl` provides the :func:`dpctl.get_devices` utility function to list +the available devices on a user's system. The list of devices returned depends +on available hardware, installed drivers, as well as by +:dpcpp_envar:`environment variables <>` influencing SYCL runtime +such as ``SYCL_DEVICE_FILTER`` or ``SYCL_DEVICE_ALLOWLIST``. + +.. _fig-listing-devices: + +.. literalinclude:: ../../../../../examples/python/device_selection.py + :language: python + :lines: 20-22, 107-131 + :caption: Listing Available Devices + :linenos: + +A possible output for the example :ref:`fig-listing-devices` may be: + +.. program-output:: python ../examples/python/device_selection.py -r list_devices + +The example :ref:`fig-listing-devices` demonstrates the usage of +:func:`dpctl.get_devices`. The list can be filtered based on +:class:`dpctl.backend` and :class:`dpctl.device_type`. The 0-based ordinal +position of a device in the output of :func:`dpctl.get_devices` corresponds to +the ``device id`` value in the filter selector string corresponding to the +device. For example, ``"opencl:cpu:0"`` refers to the first device in the list +returned by ``dpctl.get_devices(backend="opencl", device_type="cpu")``. If such +a list is empty, device construction call ``dpctl.SyclDevice("opencl:gpu:0")`` +will raise a ``ValueError``. + +.. Note:: + + Unless the system configuration changes, the list of devices returned by + :func:`dpctl.get_devices` and the relative ordering of devices in the list + is stable for every call to the function, even across different runs of an + application. + +Device Aspects and Information Descriptors +------------------------------------------ + +A device can have various *aspects* and *information descriptors* that describe +its hardware characteristics. :sycl_aspects:`Aspects <>` are boolean +characteristics of the device, whereas +:sycl_device_info:`information descriptors <>` are non-boolean characteristics +that provide more verbose information about the device. +:class:`dpctl.SyclDevice` exposes various Python properties that describe a +device's aspects and information descriptors. For example, the property +``has_aspect_fp16`` returns a boolean expression indicating whether a +particular device has aspect ``"fp16"``, indicating whether it supports the +IEEE-754 half-precision floating point type. Whereas, the ``name`` property is +an information descriptor that returns a string with the name of the device. + +.. _fig-available-properties: + +.. code-block:: Python + :caption: Listing Available Device Aspects and Information Descriptors + :linenos: + + import dpctl + import inspect + + def get_properties(cls, prop_name): + "Get name of properties of a class known to have `prop_name`" + known_property_t = type(getattr(cls, prop_name)) + return [n for n, o in inspect.getmembers(cls) if isinstance(o, known_property_t)] + + print(len(get_properties(dpctl.SyclDevice, "name"))) + # Output: 52 + +The example :ref:`fig-available-properties` demonstrates a programmatic way of +listing all the aspects and information descriptor properties in +:class:`dpctl.SyclDevice`. + +.. _sec-devices-sub-devices: + +Sub-devices +----------- + +It is possible for a device to be partitioned into "sub-devices". A sub-device +represents a sub-set of the computational units within a device that are grouped +based on some hardware criteria. For example, a two socket CPU device may be +partitioned into two sub-devices, where each sub-device represents a separate +:numa_domain:`NUMA domain <>`. Depending on the hardware characteristics and +the capabilities of the SYCL runtime, a sub-device may be partitioned further. + +For devices that support partitioning, the +:func:`dpctl.SyclDevice.create_sub_devices` can be used to create a list of +sub-devices. The requested partitioning scheme is indicated with use of the +required ``partition`` keyword. Several types of partitioning schemes are +available: + +* **Count partitioning** + The partitioning scheme is specified as a list of positive integers + indicating a partitioning with each sub-device having the requested number + of parallel compute units, or as a single positive integer indicating + equal-counts partition. + +* **Affinity partitioning** + The partitioning scheme is specified as a string indicating an affinity + domain used to create sub-devices that sharing a common resource, such as + certain hardware cache levels. + +.. Note:: + + Use ``partition="next_partitionable"`` to partition along the next level of + architectural hierarchy. + +The following example shows an affinity-based partitioning of a CPU device +into sub-devices based on the available NUMA domains. + +.. _fig-partition-cpu: + +.. literalinclude:: ../../../../../examples/python/subdevices.py + :language: python + :lines: 17, 62-76 + :caption: Partitioning a CPU device + :linenos: + +A possible output for the example :ref:`fig-partition-cpu` may be: + +.. program-output:: python ../examples/python/subdevices.py -r subdivide_by_affinity diff --git a/docs/docfiles/user_guides/manual/dpctl/intro.rst b/docs/docfiles/user_guides/manual/dpctl/intro.rst new file mode 100644 index 0000000000..327178919e --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/intro.rst @@ -0,0 +1,38 @@ +.. _intro: + +dpctl +----- + +The Data Parallel Control (dpctl) package provides a Python runtime to access a +data-parallel computing resource or *XPU* from another Python application or +library, alleviating the need for the other Python packages to develop such a +runtime themselves. The term XPU denotes a diverse range of compute +architectures such as a CPU, GPU, FPGA, *etc.*, available to programmers on a +modern heterogeneous system. + +The dpctl runtime is built on top of the C++ SYCL standard and is designed to be +both vendor and architecture agnostic. If the underlying SYCL runtime supports +a type of architecture, the dpctl runtime will allow accessing that architecture +from Python. + +In its current form, dpctl relies on certain DPC++ extensions of SYCL standard. +Moreover, the binary distribution of dpctl uses the proprietary Intel(R) oneAPI +DPC++ runtime bundled as part of oneAPI and supports Intel XPU devices only. +However, dpctl is compatible with the runtime of open-source DPC++ SYCL bundle +that can be compiled to support a wide range of architectures including CUDA, +AMD ROC, and HIP. + +The user guide introduces the core features of dpctl and the underlying +concepts. The guide is meant primarily for users of the Python package. Library +and native extension developers should refer to the programmer's guide. + +Table of contents ++++++++++++++++++ + +.. toctree:: + :maxdepth: 2 + + basic_concepts + device_selection + platforms + devices diff --git a/docs/docfiles/user_guides/manual/dpctl/platforms.rst b/docs/docfiles/user_guides/manual/dpctl/platforms.rst new file mode 100644 index 0000000000..bf9c0ed981 --- /dev/null +++ b/docs/docfiles/user_guides/manual/dpctl/platforms.rst @@ -0,0 +1,35 @@ +.. _querying_platforms: + +######## +Platform +######## + +A platform abstracts a device driver for one or more XPU that is connected to +a host. The :class:`dpctl.SyclPlatform` class represents a platform and +abstracts the :sycl_platform:`sycl::platform <>` SYCL runtime class. + +Listing Available Platforms +--------------------------- + +The platforms available on a system can be queried using the +:func:`dpctl.lsplatform` function. In addition, as illustrated in the following +example it is possible to print out metadata about a platform. + +.. literalinclude:: ../../../../../examples/python/lsplatform.py + :language: python + :lines: 20-41 + :linenos: + +The example can be executed as follows: + +.. code-block:: bash + + python dpctl/examples/python/lsplatform.py -r all + +The possible output for the example may be: + +.. program-output:: python ../examples/python/lsplatform.py -r all + +.. Note:: + The verbosity for the output can be controlled using the ``verbosity`` + keyword argument. Refer :func:`dpctl.lsplatform`. diff --git a/docs/doxyrest-config.lua.in b/docs/doxyrest-config.lua.in index 857260ddeb..42de9e7797 100644 --- a/docs/doxyrest-config.lua.in +++ b/docs/doxyrest-config.lua.in @@ -82,7 +82,7 @@ EXTRA_PAGE_LIST = {} --! is not set (otherwise, the title of intro file will be used). --! -INDEX_TITLE = "dpctl C API" +INDEX_TITLE = "libsyclinterface" --! --! File with project introduction (reStructuredText). When non-nil, this file diff --git a/docs/extlinks_gen.py b/docs/extlinks_gen.py new file mode 100644 index 0000000000..caa45a9e94 --- /dev/null +++ b/docs/extlinks_gen.py @@ -0,0 +1,36 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + + +def create_extlinks(): + """Reads a JSON file to create a dictionary of urls in the format supported + by the sphinx.ect.extlinks extension. + + Returns: + dict: A dictionary that is understood by the extlinks Sphinx extension. + + """ + extlinks = {} + + with open("docfiles/urls.json") as urls_json: + urls = json.load(urls_json) + for url in urls: + url_value = urls[url] + extlinks[url] = (url_value + "%s", None) + + return extlinks diff --git a/docs/generate_rst.py b/docs/generate_rst.py new file mode 100644 index 0000000000..1c80f4bc4a --- /dev/null +++ b/docs/generate_rst.py @@ -0,0 +1,735 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" The module provides helper functions to generate API documentation for + dpctl and its members. +""" + +import argparse +import enum +import inspect +import io +import os +import sys +from importlib import import_module +from pkgutil import iter_modules + +import dpctl + +# known property in Cython extension class +_getset_descriptor = type(dpctl.SyclDevice.name) +# known method (defined using def in Cython extension class) +_cython_method_type = type(dpctl.SyclDevice.get_filter_string) +# known builtin method (defined using cpdef in Cython extension class) +_cython_builtin_function_or_method_type = type(dpctl.SyclQueue.mro) + +# Dictionary mapping internal module names to a readable string. so that we +# can use the module name to logically group functions. +function_groups = { + "dpctl._device_selection": "Device Selection Functions", + "dpctl._sycl_queue_manager": "Queue Management Functions", + "dpctl.tensor._ctors": "Array Construction", + "dpctl.tensor._copy_utils": "Array Construction", + "dpctl.tensor._dlpack": "Array Construction", + "dpctl.tensor._reshape": "Array Manipulation", + "dpctl.memory._memory": "Functions", + "dpctl.program._program": "Functions", + "dpctl.utils._compute_follows_data": "Functions", +} + + +def _get_module(module): + """Get the Python object for a module from a string providing the module's + name. + + Args: + module ([str]): The name of a module to be searched in ``sys.modules``. + + Raises: + ValueError: If no corresponding module object was found for the string + module name. + + Returns: + [object]: A Python object representing a module. + """ + try: + return sys.modules[module] + except KeyError: + raise ValueError( + module + "is not a valid module name or it is not loaded" + ) + + +def _write_line(output, s): + """Write a line to specified out stream. + + Args: + output (``io.StringIO``): The string stream to be written. + s (str): The string that is to be written out as a line. + """ + output.write(s) + output.write("\n") + + +def _write_empty_line(output): + """[summary] + + Args: + output ([type]): [description] + """ + _write_line(output, "") + + +def _write_marquee(o, s): + """[summary] + + Args: + o ([type]): [description] + s ([type]): [description] + """ + marquee = "#" * len(s) + _write_line(o, marquee) + _write_line(o, s) + _write_line(o, marquee) + + +def _write_underlined(o, s, c): + """[summary] + + Args: + o ([type]): [description] + s ([type]): [description] + c ([type]): [description] + """ + _write_line(o, s) + _write_line(o, c * len(s)) + + +def _write_hidden_toc(o, list_of_obj_names, prefix_str="", suffix_str=""): + """[summary] + + Args: + o ([type]): [description] + list_of_objs ([type]): [description] + prefix_str ([type]): [description] + suffix_str ([type]): [description] + """ + if not list_of_obj_names: + return + _write_line(o, ".. toctree::") + _write_line(o, " :hidden:") + _write_empty_line(o) + for obj in list_of_obj_names: + _write_line(o, " " + prefix_str + obj + suffix_str) + _write_empty_line(o) + + +def _get_public_class_name(cls): + """[summary] + + Raises: + TypeError: [description] + + Returns: + [type]: [description] + """ + if not inspect.isclass(cls): + raise TypeError("Expecting class, got {}".format(type(cls))) + modl = cls.__module__ + if modl: + modl = ".".join( + [comp for comp in modl.split(".") if not comp.startswith("_")] + ) + if modl: + res = ".".join([modl, cls.__qualname__]) + else: + res = cls.__qualname__ + return res + + +def _is_class_property(o): + """[summary] + + Args: + o ([type]): [description] + + Returns: + [type]: [description] + """ + return isinstance(o, property) or (type(o) == _getset_descriptor) + + +def _is_class_method(o): + """[summary] + + Args: + o ([type]): [description] + + Returns: + [type]: [description] + """ + return inspect.ismethod(o) or ( + type(o) + in [_cython_method_type, _cython_builtin_function_or_method_type] + ) + + +def _get_filtered_names(cls, selector_func): + """[summary] + + Args: + selector_func ([type]): [description] + + Returns: + [type]: [description] + """ + return [ + _name + for _name, _obj in inspect.getmembers(cls, selector_func) + if not _name.startswith("__") + ] + + +def _group_functions(mod): + """Bin module functions into a set of logical groups. + + Args: + mod (object): A module whose functions will be grouped into bins + based on the ``function_groups`` dictionary. + + Returns: + [dict]: A dictionary containing grouping of functions in the + module. + """ + groups = {} + for name, obj in inspect.getmembers(mod): + if inspect.isbuiltin(obj) or inspect.isfunction(obj): + if obj.__module__ and obj.__module__ in function_groups: + try: + flist = groups[function_groups[obj.__module__]] + flist.append(obj) + except KeyError: + groups[function_groups[obj.__module__]] = [ + obj, + ] + else: + # Special case for _sycl_device_factory + if ( + obj.__module__ == "dpctl._sycl_device_factory" + and "select_" in obj.__name__ + ): + try: + flist = groups["Device Selection Functions"] + flist.append(obj) + except KeyError: + groups["Device Selection Functions"] = [ + obj, + ] + else: + try: + flist = groups["Other Functions"] + flist.append(obj) + except KeyError: + groups["Other Functions"] = [ + obj, + ] + return groups + + +def _generate_class_rst(cls): + """Generate a rst file with the API documentation for a class. + + Raises: + TypeError: When the input is not a Python class + + Returns: + [str]: A string with rst nodes that can be written out to a file. + """ + if not inspect.isclass(cls): + raise TypeError("Expecting class, got {}".format(type(cls))) + + cls_qualname = _get_public_class_name(cls) + rst_header = cls_qualname.split(".")[-1] + rst_module = ".".join(cls_qualname.split(".")[:-1]) + rst_header = "".join([".. _", rst_header, "_api:"]) + + def write_rubric(o, indent, rubric_display, rubric_tag, cls_qualname): + _write_line(o, indent + ".. rubric:: " + rubric_display) + _write_empty_line(o) + _write_line(o, indent + ".. autoautosummary:: " + cls_qualname) + _write_line(o, indent + indent + ":" + rubric_tag + ":") + _write_empty_line(o) + + with io.StringIO() as output: + # Attributes + all_attributes = _get_filtered_names(cls, _is_class_property) + # Methods, separated into public/private + all_methods = _get_filtered_names(cls, _is_class_method) + all_public_methods = [] + all_private_methods = [] + for _name in all_methods: + if _name.startswith("_"): + all_private_methods.append(_name) + else: + all_public_methods.append(_name) + + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, cls_qualname) + _write_empty_line(output) + + _write_line(output, ".. currentmodule:: " + rst_module) + _write_empty_line(output) + + _write_line(output, ".. autoclass:: " + cls_qualname) + _write_empty_line(output) + + indent = " " + attributes_header = "Attributes" + private_methods_header = "Private methods" + public_methods_header = "Public methods" + + if all_attributes: + write_rubric( + output, + indent, + attributes_header + ":", + "attributes", + cls_qualname, + ) + if all_public_methods: + write_rubric( + output, + indent, + public_methods_header + ":", + "methods", + cls_qualname, + ) + if all_private_methods: + write_rubric( + output, + indent, + private_methods_header + ":", + "private_methods", + cls_qualname, + ) + + _write_empty_line(output) + + if all_attributes: + _write_underlined(output, attributes_header, "-") + _write_empty_line(output) + for n in all_attributes: + _write_line( + output, + ".. autoattribute:: " + ".".join([cls_qualname, n]), + ) + _write_empty_line(output) + + if all_public_methods: + _write_underlined(output, public_methods_header, "-") + _write_empty_line(output) + for n in all_public_methods: + _write_line( + output, + ".. autofunction:: " + ".".join([cls_qualname, n]), + ) + _write_empty_line(output) + + # Private methods + if all_private_methods: + _write_underlined(output, private_methods_header, "-") + _write_empty_line(output) + for n in all_private_methods: + _write_line( + output, + ".. autofunction:: " + ".".join([cls_qualname, n]), + ) + return output.getvalue() + + +def _generate_module_summary_rst(module): + """[summary] + + Args: + module ([str]): [description] + + Returns: + [type]: [description] + """ + rst_header = "".join([".. _", module, "_pyapi:"]) + pagename = module + indent = " " + + def _get_doc_summary(obj): + docstr = getattr(obj, "__doc__") + if not isinstance(docstr, str): + docstr = f"[FIXME]: {type(obj)} does not have a docstring" + return docstr + # Let us stip out any newlines, tabs, etc. at the start of the docstr + docstr = docstr.lstrip() + # Check if a signature line is provided. The check only looks for + # something like "SyclContext(" + st = ( + len(obj.__name__ + "(") + if docstr.startswith(obj.__name__ + "(", 0) + else 0 + ) + # If an apparent signature line was seen, then locate the end of the + # signature line. + if st: + nOpens = 1 + for i, c in enumerate(docstr[st:]): + if c == "(": + nOpens += 1 + elif c == ")": + stop = i + nOpens -= 1 + if nOpens == 0: + break + st += stop + 1 + # Strip out the signature in the docstring. + docstr = docstr[st:] + # The hope is to find the first line (summary) from the docstring + # by searching for a period followed by a new line. Not foolproof, but + # a best-effort check. + docstr = " ".join( + docstr[0 : docstr.find(".\n") + 1].replace("\n", " ").split() + ) + if not docstr: + return f"[FIXME]: {type(obj)} has a docstring with no summary" + return docstr + + def _write_table_header(o): + _write_line(o, ".. list-table::") + _write_line(o, indent + ":widths: 25,50") + _write_empty_line(o) + + def _write_submodules_summary_table(o, mod): + submods = [ + submod.name for submod in iter_modules(mod.__path__) if submod.ispkg + ] + if submods: + _write_empty_line(output) + _write_underlined(output, "Sub-modules", "-") + _write_empty_line(output) + _write_hidden_toc(output, submods, mod.__name__ + ".", "_pyapi") + _write_table_header(o) + for submod in submods: + _write_line( + o, + indent + + "* - :ref:`" + + mod.__name__ + + "." + + submod + + "_pyapi`", + ) + _submod = import_module(module + "." + submod, mod.__name__) + mod_summary = _get_doc_summary(_submod) + _write_line(o, indent + " - " + mod_summary) + _write_empty_line(o) + + def _write_classes_summary_table(o, mod): + classes = [] + class_names = [] + for mem_tup in inspect.getmembers(mod): + cls = mem_tup[1] + if inspect.isclass(cls) and not ( + issubclass(cls, enum.Enum) or issubclass(cls, Exception) + ): + classes.append(cls) + class_names.append(mem_tup[0]) + if classes: + _write_line(o, ".. _" + mod.__name__.lower() + "_classes:") + _write_empty_line(o) + _write_underlined(o, "Classes", "-") + _write_empty_line(o) + _write_hidden_toc(output, class_names) + _write_table_header(o) + for cls in classes: + _write_line(o, indent + "* - :class:`" + cls.__name__ + "`") + # For classes, the first line of the docstring is the + # signature. So we skip that line to pick up the summary. + cls_summary = _get_doc_summary(cls) + _write_line(o, indent + " - " + cls_summary) + _write_empty_line(o) + + def _write_enums_summary_table(o, mod): + enums = [] + for mem_tup in inspect.getmembers(mod): + e = mem_tup[1] + if inspect.isclass(e) and (issubclass(e, enum.Enum)): + enums.append(e) + if enums: + _write_underlined(o, "Enums", "-") + _write_empty_line(o) + _write_table_header(o) + for e in enums: + _write_line(o, indent + "* - :class:`" + e.__name__ + "`") + enum_summary = _get_doc_summary(e) + _write_line(o, indent + " - " + enum_summary) + _write_empty_line(o) + + def _write_exceptions_summary_table(o, mod): + exps = [] + for mem_tup in inspect.getmembers(mod): + e = mem_tup[1] + if inspect.isclass(e) and (issubclass(e, Exception)): + exps.append(e) + + if exps: + _write_underlined(o, "Exceptions", "-") + _write_empty_line(o) + _write_table_header(o) + for e in exps: + _write_line(o, indent + "* - :class:`" + e.__name__ + "`") + excep_summary = _get_doc_summary(e) + _write_line(o, indent + " - " + excep_summary) + _write_empty_line(o) + + def _write_functions_summary_table(o, mod, fnobj_list): + _write_table_header(o) + for fnobj in fnobj_list: + _write_line(o, indent + "* - :func:`" + fnobj.__name__ + "()`") + # For functions, the first line of the docstring is the + # signature. So we skip that line to pick up the summary. + fn_summary = _get_doc_summary(fnobj) + _write_line(o, indent + " - " + fn_summary) + _write_empty_line(o) + + def _write_function_groups_summary(o, mod, groups): + + for group in groups: + if group != "Other Functions": + _write_line( + o, + ".. _" + + mod.__name__.lower() + + "_" + + group.lower().replace(" ", "_") + + ":", + ) + _write_empty_line(o) + _write_underlined(o, group, "-") + _write_empty_line(o) + _write_functions_summary_table(o, mod, groups[group]) + + # We want to write "Other Functions" in the end always + try: + other_fns = groups["Other Functions"] + _write_line( + o, + ".. _" + mod.__name__.lower() + "_other_functions:", + ) + _write_empty_line(o) + _write_underlined(o, "Other Functions", "-") + _write_empty_line(o) + _write_functions_summary_table(o, mod, other_fns) + except KeyError: + pass + + mod = _get_module(module) + + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + _write_line(output, ".. automodule:: " + module) + _write_empty_line(output) + _write_submodules_summary_table(output, mod) + _write_empty_line(output) + _write_classes_summary_table(output, mod) + _write_empty_line(output) + _write_function_groups_summary(output, mod, _group_functions(mod)) + _write_empty_line(output) + _write_enums_summary_table(output, mod) + _write_empty_line(output) + _write_exceptions_summary_table(output, mod) + _write_empty_line(output) + + return output.getvalue() + + +def _generate_rst_for_all_classes(module, outputpath): + """Generates rst API docs for all classes in a module and writes them to + given path. + + Args: + module ([str]): Name of module that needs to be documented + outputpath ([str]): Path where the rst files are to be saved. + """ + mod = _get_module(module) + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and not ( + issubclass(obj, enum.Enum) or issubclass(obj, Exception) + ): + out = outputpath + "/" + name + ".rst" + with open(out, "w") as rst_file: + rst_file.write(_generate_class_rst(obj)) + + +def _generate_rst_for_all_functions(module, outputpath): + """[summary] + + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ + mod = _get_module(module) + groups = _group_functions(mod) + + rst_header = "".join([".. _", module, "_functions_api:"]) + pagename = module + " Functions" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + def _write_function_autodocs(o, groups): + for group, fnlist in groups.items(): + _write_empty_line(o) + _write_underlined(o, group, "-") + _write_empty_line(o) + for fn in fnlist: + _write_line(output, ".. autofunction:: " + fn.__name__) + + out = outputpath + "/" + module + "_functions_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + _write_function_autodocs(output, groups) + rst_file.write(output.getvalue()) + + +def _generate_rst_for_all_exceptions(module, outputpath): + """[summary] + + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ + mod = _get_module(module) + rst_header = "".join([".. _", module, "_exception_api:"]) + pagename = module + " Exceptions" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + out = outputpath + "/" + module + "_exception_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, Exception): + _write_line(output, ".. autoexception:: " + obj.__name__) + + rst_file.write(output.getvalue()) + + +def _generate_rst_for_all_enums(module, outputpath): + """[summary] + + Args: + module ([type]): [description] + outputpath ([type]): [description] + + Raises: + ValueError: [description] + """ + mod = _get_module(module) + indent = " " + rst_header = "".join([".. _", module, "_enum_api:"]) + pagename = module + " Enums" + + if not os.path.exists(outputpath): + raise ValueError("Invalid output path provided") + + out = outputpath + "/" + module + "_enum_api.rst" + with open(out, "w") as rst_file: + with io.StringIO() as output: + _write_line(output, rst_header) + _write_empty_line(output) + _write_marquee(output, pagename) + _write_empty_line(output) + _write_empty_line(output) + _write_line(output, ".. currentmodule:: " + module) + _write_empty_line(output) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and issubclass(obj, enum.Enum): + _write_line(output, ".. autoclass:: " + obj.__name__) + _write_line(output, indent + ":members:") + + rst_file.write(output.getvalue()) + + +def generate_all(module, outputpath): + """Recursively generate rst files for a root module and all its members. + + Args: + module ([str]): Name of a Python module + outputpath ([str]): Output directory + """ + mod = _get_module(module) + out = outputpath + "/" + module + "_pyapi.rst" + # Generate a summary page for the module's API + with open(out, "w") as rst_file: + rst_file.write(_generate_module_summary_rst(module)) + # Generate supporting pages for the module + _generate_rst_for_all_classes(module, outputpath) + _generate_rst_for_all_enums(module, outputpath) + _generate_rst_for_all_exceptions(module, outputpath) + _generate_rst_for_all_functions(module, outputpath) + + # Now recurse into any submodule and generate all for them too. + for submod in iter_modules(mod.__path__): + if submod.ispkg: + generate_all(module + "." + submod.name, outputpath) + + +parser = argparse.ArgumentParser("Generate rst files for Python source files") +parser.add_argument("--dir", help="Output directory", required=True) +parser.add_argument("--module", help="Python module", required=True) + +args = parser.parse_args() +outdir = args.dir +mod = args.module + +# Run generate_all +generate_all(mod, outdir) diff --git a/docs/index_doxyrest.rst.in b/docs/index_doxyrest.rst.in index 51e55562f9..9e064ed047 100644 --- a/docs/index_doxyrest.rst.in +++ b/docs/index_doxyrest.rst.in @@ -1,16 +1,21 @@ .. include:: ./docfiles/intro.rst + +How-to Guides +============= + .. toctree:: :maxdepth: 1 - :caption: User Guides docfiles/user_guides/QuickStart + docfiles/user_guides/UserManual + .. toctree:: - :maxdepth: 1 - :caption: API Documentation + :maxdepth: 1 + :caption: API Documentation - docfiles/dpctl_pyapi - @DOXYREST_OUTPUT_DIR_NAME@/index + docfiles/dpctl/dpctl_pyapi + @DOXYREST_OUTPUT_DIR_NAME@/index .. include:: ./docfiles/boilerplate.rst diff --git a/docs/index_no_doxyrest.rst.in b/docs/index_no_doxyrest.rst.in index fffbf63cb4..ac57d680ae 100644 --- a/docs/index_no_doxyrest.rst.in +++ b/docs/index_no_doxyrest.rst.in @@ -1,15 +1,19 @@ .. include:: ./docfiles/intro.rst +How-to Guides +============= + .. toctree:: :maxdepth: 1 - :caption: User Guides docfiles/user_guides/QuickStart + docfiles/user_guides/UserManual + .. toctree:: - :maxdepth: 1 - :caption: API Documentation + :maxdepth: 1 + :caption: API Documentation - docfiles/dpctl_pyapi + docfiles/dpctl/dpctl_pyapi .. include:: ./docfiles/boilerplate.rst diff --git a/dpctl-capi/helper/source/dpctl_async_error_handler.cpp b/dpctl-capi/helper/source/dpctl_async_error_handler.cpp deleted file mode 100644 index 6b5d6db191..0000000000 --- a/dpctl-capi/helper/source/dpctl_async_error_handler.cpp +++ /dev/null @@ -1,42 +0,0 @@ -//===-- dpctl_async_error_handler.h - An async error handler -*-C++-*- ===// -// -// Data Parallel Control (dpctl) -// -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// A functor to use for passing an error handler callback function to sycl -/// context and queue contructors. -//===----------------------------------------------------------------------===// - -#include "dpctl_async_error_handler.h" - -void DPCTL_AsyncErrorHandler::operator()( - const cl::sycl::exception_list &exceptions) -{ - for (std::exception_ptr const &e : exceptions) { - try { - std::rethrow_exception(e); - } catch (cl::sycl::exception const &e) { - std::cerr << "Caught asynchronous SYCL exception:\n" - << e.what() << std::endl; - // FIXME: Change get_cl_code() to code() once DPCPP supports it. - auto err_code = e.get_cl_code(); - handler_(err_code); - } - } -} diff --git a/dpctl-capi/tests/dpcpp_kernels.cpp b/dpctl-capi/tests/dpcpp_kernels.cpp deleted file mode 100644 index 897aa86de4..0000000000 --- a/dpctl-capi/tests/dpcpp_kernels.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "dpcpp_kernels.hpp" -#include -#include - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, int *, int); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, - size_t, - unsigned int *, - unsigned int); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, double *, double); - -template sycl::kernel -dpcpp_kernels::get_fill_kernel(sycl::queue &, size_t, float *, float); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, int *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, - size_t, - unsigned int *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, float *); - -template sycl::kernel -dpcpp_kernels::get_range_kernel(sycl::queue &, size_t, double *); - -template sycl::kernel dpcpp_kernels::get_mad_kernel(sycl::queue &, - size_t, - int *, - int *, - int *, - int); - -template sycl::kernel -dpcpp_kernels::get_mad_kernel(sycl::queue &, - size_t, - unsigned int *, - unsigned int *, - unsigned int *, - unsigned int); - -template sycl::kernel dpcpp_kernels::get_local_sort_kernel(sycl::queue &, - size_t, - size_t, - int *, - size_t); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - int *, - size_t, - int, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - unsigned int *, - size_t, - unsigned int, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - float *, - size_t, - float, - int *); - -template sycl::kernel -dpcpp_kernels::get_local_count_exceedance_kernel(sycl::queue &, - size_t, - size_t, - double *, - size_t, - double, - int *); diff --git a/dpctl/.gitignore b/dpctl/.gitignore index 0c03f1ed6a..3e23a8af25 100644 --- a/dpctl/.gitignore +++ b/dpctl/.gitignore @@ -1,5 +1,6 @@ *.so *.cpp +*.cxx *.c *.h memory/*.h diff --git a/dpctl/CMakeLists.txt b/dpctl/CMakeLists.txt new file mode 100644 index 0000000000..541ef7d5e8 --- /dev/null +++ b/dpctl/CMakeLists.txt @@ -0,0 +1,192 @@ + +find_package(PythonExtensions REQUIRED) +find_package(NumPy REQUIRED) + +set(CYTHON_FLAGS "-t -w ${CMAKE_SOURCE_DIR}") +find_package(Cython REQUIRED) + +if(WIN32) + string(CONCAT WARNING_FLAGS + "-Wall " + "-Wextra " + "-Winit-self " + "-Wunused-function " + "-Wuninitialized " + "-Wmissing-declarations " + "-Wno-unused-parameter " + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Ox ${WARNING_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Ox ${WARNING_FLAGS}") + set(CMAKE_C_FLAGS_DEBUG + "${CMAKE_C_FLAGS_DEBUG} ${WARNING_FLAGS} -ggdb3 -DDEBUG" + ) + set(CMAKE_CXX_FLAGS_DEBUG + "${CMAKE_CXX_FLAGS_DEBUG} ${WARNING_FLAGS} -ggdb3 -DDEBUG" + ) + set(DPCTL_LDFLAGS "/NXCompat /DynamicBase") +elseif(UNIX) + string(CONCAT WARNING_FLAGS + "-Wall " + "-Wextra " + "-Winit-self " + "-Wunused-function " + "-Wuninitialized " + "-Wmissing-declarations " + "-fdiagnostics-color=auto " + "-Wno-deprecated-declarations " + ) + string(CONCAT SDL_FLAGS + "-fstack-protector " + "-fstack-protector-all " + "-fpic " + "-fPIC " + "-D_FORTIFY_SOURCE=2 " + "-Wformat " + "-Wformat-security " + "-fno-strict-overflow " + "-fno-delete-null-pointer-checks " + ) + string(CONCAT CFLAGS + "${WARNING_FLAGS}" + "${SDL_FLAGS}" + ) + string(CONCAT CXXFLAGS + "${WARNING_FLAGS}" + "${SDL_FLAGS}" + "-fsycl " + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 ${CFLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 ${CXXFLAGS}") + set(CMAKE_C_FLAGS_DEBUG + "${CMAKE_C_FLAGS_DEBUG} ${CFLAGS} -ggdb3 -DDEBUG" + ) + set(CMAKE_CXX_FLAGS_DEBUG + "${CMAKE_CXX_FLAGS_DEBUG} ${CXXFLAGS} -ggdb3 -DDEBUG" + ) + set(DPCTL_LDFLAGS "-z,noexecstack,-z,relro,-z,now") +else() + message(FATAL_ERROR "Unsupported system.") +endif() + +# at build time create include/ directory and copy header files over +set(DPCTL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) +add_custom_target(_build_time_create_dpctl_include ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface/Support + COMMAND ${CMAKE_COMMAND} -E make_directory ${DPCTL_INCLUDE_DIR}/syclinterface/Config + DEPENDS DPCTLSyclInterface +) + +set(_copied_header_files) +file(GLOB _syclinterface_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/*.h) +foreach(hf ${_syclinterface_h}) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + VERBATIM + ) +endforeach() + +file(GLOB _syclinterface_Support_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Support/*.h) +foreach(hf ${_syclinterface_Support_h}) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/Support/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) +endforeach() + +file(GLOB _syclinterface_Config_h ${CMAKE_SOURCE_DIR}/libsyclinterface/include/Config/*.h) +foreach(hf ${_syclinterface_Config_h}) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/syclinterface/Config/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) +endforeach() + +file(GLOB _apis_h ${CMAKE_CURRENT_SOURCE_DIR}/apis/include/*) +foreach(hf ${_apis_h}) + get_filename_component(_header_name ${hf} NAME) + set(_target_header_file ${DPCTL_INCLUDE_DIR}/${_header_name}) + list(APPEND _copied_header_files ${_target_header_file}) + add_custom_command(OUTPUT ${_target_header_file} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${hf} ${_target_header_file} + DEPENDS ${hf} _build_time_create_dpctl_include + ) +endforeach() + +add_custom_target( + _build_time_create_dpctl_include_copy ALL + DEPENDS ${_copied_header_files} +) + +set(CMAKE_INSTALL_RPATH "$ORIGIN") + +function(build_dpctl_ext _trgt _src _dest) + add_cython_target(${_trgt} ${_src} CXX OUTPUT_VAR _generated_src) + add_library(${_trgt} MODULE ${_generated_src}) + target_include_directories(${_trgt} PRIVATE ${NumPy_INCLUDE_DIR} ${DPCTL_INCLUDE_DIR}) + add_dependencies(${_trgt} _build_time_create_dpctl_include_copy) + if (DPCTL_GENERATE_COVERAGE) + target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1) + target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer) + endif() + target_link_libraries(${_trgt} DPCTLSyclInterface) + target_link_options(${_trgt} PRIVATE "LINKER:${DPCTL_LDFLAGS}") + python_extension_module(${_trgt}) + get_filename_component(_name_wle ${_generated_src} NAME_WLE) + get_filename_component(_generated_src_dir ${_generated_src} DIRECTORY) + set(_generated_public_h "${_generated_src_dir}/${_name_wle}.h") + set(_generated_api_h "${_generated_src_dir}/${_name_wle}_api.h") + set(_copy_trgt "${_trgt}_copy_capi_include") + add_custom_target( + ${_copy_trgt} ALL + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_public_h} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_api_h} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake + DEPENDS ${_trgt} + VERBATIM + COMMENT "Copying Cython-generated headers to dpctl" + ) + if (DPCTL_GENERATE_COVERAGE) + set(_copy_cxx_trgt "${_trgt}_copy_cxx") + add_custom_target( + ${_copy_cxx_trgt} ALL + COMMAND ${CMAKE_COMMAND} + -DSOURCE_FILE=${_generated_src} + -DDEST=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_SOURCE_DIR}/dpctl/cmake/copy_existing.cmake + DEPENDS ${_trgt} + VERBATIM + COMMENT "Copying Cython-generated source to dpctl" + ) + endif() + install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest}) +endfunction() + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl") +endforeach() + +target_include_directories(_sycl_queue PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + +add_subdirectory(program) +add_subdirectory(memory) +add_subdirectory(tensor) +add_subdirectory(utils) diff --git a/dpctl/__init__.py b/dpctl/__init__.py index efd93f3630..ca4ee9796f 100644 --- a/dpctl/__init__.py +++ b/dpctl/__init__.py @@ -18,11 +18,12 @@ **Data Parallel Control (dpctl)** is a Python abstraction layer over SYCL. Dpctl implements a subset of SYCL's API providing wrappers for the - SYCL runtime classes described in `Section 4.6`_ of the `SYCL 2020 spec`_. - Note that the SYCL ``device_selector`` class is not implemented, instead - there are device selection helper functions that can be used to simulate - the same behavior. Dpctl implements the ``ONEPI::filter_selector`` extension - that is included in Intel's DPC++ SYCL compiler. + SYCL runtime classes described in :sycl_runtime_classes:`Section 4.6 <>` of + the :sycl_spec_2020:`SYCL 2020 spec <>`. Note that the SYCL + ``device_selector`` class is not implemented, instead there are device + selection helper functions that can be used to simulate the same behavior. + Dpctl implements the ``ONEPI::filter_selector`` extension that is included + in Intel's DPC++ SYCL compiler. The module also includes a global SYCL queue manager. The queue manager provides convenience functions to create a global instance of @@ -123,11 +124,18 @@ __all__ += [ "get_include", ] +# add submodules +__all__ += [ + "memory", + "program", + "tensor", + "utils", +] def get_include(): - """ - Return the directory that contains the dpctl *.h header files. + r""" + Return the directory that contains the dpctl \*.h header files. Extension modules that need to be compiled against dpctl should use this function to locate the appropriate include directory. diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index eefdd4da85..8a666b045b 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -25,15 +25,15 @@ from libc.stdint cimport int64_t, uint32_t from libcpp cimport bool -cdef extern from "dpctl_error_handler_type.h": +cdef extern from "syclinterface/dpctl_error_handler_type.h": ctypedef void error_handler_callback(int err_code) -cdef extern from "dpctl_utils.h": +cdef extern from "syclinterface/dpctl_utils.h": cdef void DPCTLCString_Delete(const char *str) cdef void DPCTLSize_t_Array_Delete(size_t *arr) -cdef extern from "dpctl_sycl_enum_types.h": +cdef extern from "syclinterface/dpctl_sycl_enum_types.h": ctypedef enum _backend_type 'DPCTLSyclBackendType': _ALL_BACKENDS 'DPCTL_ALL_BACKENDS' _CUDA 'DPCTL_CUDA' @@ -93,7 +93,7 @@ cdef extern from "dpctl_sycl_enum_types.h": _usm_host_allocations 'usm_host_allocations', _usm_shared_allocations 'usm_shared_allocations', _usm_restricted_shared_allocations 'usm_restricted_shared_allocations', - _usm_system_allocator 'usm_system_allocator' + _usm_system_allocations 'usm_system_allocations' ctypedef enum _partition_affinity_domain_type 'DPCTLPartitionAffinityDomainType': _not_applicable 'not_applicable', @@ -111,7 +111,7 @@ cdef extern from "dpctl_sycl_enum_types.h": _COMPLETE 'DPCTL_COMPLETE' -cdef extern from "dpctl_sycl_types.h": +cdef extern from "syclinterface/dpctl_sycl_types.h": cdef struct DPCTLOpaqueSyclContext cdef struct DPCTLOpaqueSyclDevice cdef struct DPCTLOpaqueSyclDeviceSelector @@ -133,12 +133,12 @@ cdef extern from "dpctl_sycl_types.h": ctypedef DPCTLOpaqueSyclUSM *DPCTLSyclUSMRef -cdef extern from "dpctl_sycl_device_manager.h": +cdef extern from "syclinterface/dpctl_sycl_device_manager.h": cdef struct DPCTLDeviceVector ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef -cdef extern from "dpctl_sycl_device_interface.h": +cdef extern from "syclinterface/dpctl_sycl_device_interface.h": cdef bool DPCTLDevice_AreEq(const DPCTLSyclDeviceRef DRef1, const DPCTLSyclDeviceRef DRef2) cdef DPCTLSyclDeviceRef DPCTLDevice_Copy(const DPCTLSyclDeviceRef DRef) @@ -192,7 +192,7 @@ cdef extern from "dpctl_sycl_device_interface.h": cdef DPCTLSyclDeviceRef DPCTLDevice_GetParentDevice(const DPCTLSyclDeviceRef DRef) -cdef extern from "dpctl_sycl_device_manager.h": +cdef extern from "syclinterface/dpctl_sycl_device_manager.h": cdef DPCTLDeviceVectorRef DPCTLDeviceVector_CreateFromArray( size_t nelems, DPCTLSyclDeviceRef *elems) @@ -213,7 +213,7 @@ cdef extern from "dpctl_sycl_device_manager.h": cdef int64_t DPCTLDeviceMgr_GetRelativeId(const DPCTLSyclDeviceRef DRef) -cdef extern from "dpctl_sycl_device_selector_interface.h": +cdef extern from "syclinterface/dpctl_sycl_device_selector_interface.h": DPCTLSyclDeviceSelectorRef DPCTLAcceleratorSelector_Create() DPCTLSyclDeviceSelectorRef DPCTLDefaultSelector_Create() DPCTLSyclDeviceSelectorRef DPCTLCPUSelector_Create() @@ -224,11 +224,11 @@ cdef extern from "dpctl_sycl_device_selector_interface.h": int DPCTLDeviceSelector_Score(DPCTLSyclDeviceSelectorRef, DPCTLSyclDeviceRef) -cdef extern from "dpctl_sycl_event_interface.h": +cdef extern from "syclinterface/dpctl_sycl_event_interface.h": cdef DPCTLSyclEventRef DPCTLEvent_Create() cdef DPCTLSyclEventRef DPCTLEvent_Copy(const DPCTLSyclEventRef ERef) - cdef void DPCTLEvent_Wait(DPCTLSyclEventRef ERef) - cdef void DPCTLEvent_WaitAndThrow(DPCTLSyclEventRef ERef) + cdef void DPCTLEvent_Wait(DPCTLSyclEventRef ERef) nogil + cdef void DPCTLEvent_WaitAndThrow(DPCTLSyclEventRef ERef) nogil cdef void DPCTLEvent_Delete(DPCTLSyclEventRef ERef) cdef _event_status_type DPCTLEvent_GetCommandExecutionStatus(DPCTLSyclEventRef ERef) cdef _backend_type DPCTLEvent_GetBackend(DPCTLSyclEventRef ERef) @@ -246,13 +246,13 @@ cdef extern from "dpctl_sycl_event_interface.h": cdef size_t DPCTLEvent_GetProfilingInfoEnd(DPCTLSyclEventRef ERef) -cdef extern from "dpctl_sycl_kernel_interface.h": +cdef extern from "syclinterface/dpctl_sycl_kernel_interface.h": cdef const char* DPCTLKernel_GetFunctionName(const DPCTLSyclKernelRef KRef) cdef size_t DPCTLKernel_GetNumArgs(const DPCTLSyclKernelRef KRef) cdef void DPCTLKernel_Delete(DPCTLSyclKernelRef KRef) -cdef extern from "dpctl_sycl_platform_manager.h": +cdef extern from "syclinterface/dpctl_sycl_platform_manager.h": cdef struct DPCTLPlatformVector ctypedef DPCTLPlatformVector *DPCTLPlatformVectorRef @@ -265,7 +265,7 @@ cdef extern from "dpctl_sycl_platform_manager.h": cdef void DPCTLPlatformMgr_PrintInfo(const DPCTLSyclPlatformRef, size_t) -cdef extern from "dpctl_sycl_platform_interface.h": +cdef extern from "syclinterface/dpctl_sycl_platform_interface.h": cdef DPCTLSyclPlatformRef DPCTLPlatform_Copy(const DPCTLSyclPlatformRef) cdef DPCTLSyclPlatformRef DPCTLPlatform_Create() cdef DPCTLSyclPlatformRef DPCTLPlatform_CreateFromSelector( @@ -278,14 +278,14 @@ cdef extern from "dpctl_sycl_platform_interface.h": cdef DPCTLPlatformVectorRef DPCTLPlatform_GetPlatforms() -cdef extern from "dpctl_sycl_context_interface.h": +cdef extern from "syclinterface/dpctl_sycl_context_interface.h": cdef DPCTLSyclContextRef DPCTLContext_Create( const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclContextRef DPCTLContext_CreateFromDevices( const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclContextRef DPCTLContext_Copy( const DPCTLSyclContextRef CRef) @@ -299,7 +299,7 @@ cdef extern from "dpctl_sycl_context_interface.h": cdef void DPCTLContext_Delete(DPCTLSyclContextRef CtxRef) -cdef extern from "dpctl_sycl_program_interface.h": +cdef extern from "syclinterface/dpctl_sycl_program_interface.h": cdef DPCTLSyclProgramRef DPCTLProgram_CreateFromSpirv( const DPCTLSyclContextRef Ctx, const void *IL, @@ -317,13 +317,13 @@ cdef extern from "dpctl_sycl_program_interface.h": cdef void DPCTLProgram_Delete(DPCTLSyclProgramRef PRef) -cdef extern from "dpctl_sycl_queue_interface.h": +cdef extern from "syclinterface/dpctl_sycl_queue_interface.h": cdef bool DPCTLQueue_AreEq(const DPCTLSyclQueueRef QRef1, const DPCTLSyclQueueRef QRef2) cdef DPCTLSyclQueueRef DPCTLQueue_Create( const DPCTLSyclContextRef CRef, const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) cdef DPCTLSyclQueueRef DPCTLQueue_CreateForDevice( const DPCTLSyclDeviceRef dRef, @@ -356,7 +356,7 @@ cdef extern from "dpctl_sycl_queue_interface.h": size_t NDims, const DPCTLSyclEventRef *DepEvents, size_t NDepEvents) - cdef void DPCTLQueue_Wait(const DPCTLSyclQueueRef QRef) + cdef void DPCTLQueue_Wait(const DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclEventRef DPCTLQueue_Memcpy( const DPCTLSyclQueueRef Q, void *Dest, @@ -381,7 +381,7 @@ cdef extern from "dpctl_sycl_queue_interface.h": cdef bool DPCTLQueue_HasEnableProfiling(const DPCTLSyclQueueRef QRef) -cdef extern from "dpctl_sycl_queue_manager.h": +cdef extern from "syclinterface/dpctl_sycl_queue_manager.h": cdef DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() cdef bool DPCTLQueueMgr_GlobalQueueIsCurrent() cdef bool DPCTLQueueMgr_IsCurrentQueue(const DPCTLSyclQueueRef QRef) @@ -391,26 +391,28 @@ cdef extern from "dpctl_sycl_queue_manager.h": cdef size_t DPCTLQueueMgr_GetQueueStackSize() -cdef extern from "dpctl_sycl_usm_interface.h": +cdef extern from "syclinterface/dpctl_sycl_usm_interface.h": cdef DPCTLSyclUSMRef DPCTLmalloc_shared( size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLmalloc_host( size_t size, - DPCTLSyclQueueRef QRef) - cdef DPCTLSyclUSMRef DPCTLmalloc_device(size_t size, DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil + cdef DPCTLSyclUSMRef DPCTLmalloc_device( + size_t size, + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_shared( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_host( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef DPCTLSyclUSMRef DPCTLaligned_alloc_device( size_t alignment, size_t size, - DPCTLSyclQueueRef QRef) + DPCTLSyclQueueRef QRef) nogil cdef void DPCTLfree_with_queue( DPCTLSyclUSMRef MRef, DPCTLSyclQueueRef QRef) diff --git a/dpctl/_diagnostics.pyx b/dpctl/_diagnostics.pyx new file mode 100644 index 0000000000..dc98cb29db --- /dev/null +++ b/dpctl/_diagnostics.pyx @@ -0,0 +1,80 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +""" Implements developer utilities. +""" +import contextlib +import os + + +cdef extern from "syclinterface/dpctl_service.h": + cdef void DPCTLService_InitLogger(const char *, const char *) + cdef void DPCTLService_ShutdownLogger() + + +def _init_logger(log_dir=None): + """Initialize logger to use given directory to save logs. + + The call has no effect if `dpctl` was not built to use logger. + """ + cdef bytes p = b"" + cdef const char *app_name = "dpctl" + cdef char *ld_cstr = NULL + if log_dir: + if not os.path.exists(log_dir): + raise ValueError(f"Path {log_dir} does not exist") + if isinstance(log_dir, str): + p = bytes(log_dir, "utf-8") + else: + p = bytes(log_dir) + ld_cstr = p + DPCTLService_InitLogger(app_name, ld_cstr) + + +def _shutdown_logger(): + """Finalize logger. + + The call has no effect if `dpctl` was not built to use logger. + """ + DPCTLService_ShutdownLogger() + + +@contextlib.contextmanager +def syclinterface_diagnostics(verbosity="warning", log_dir=None): + """Context manager that activate verbosity of DPCTLSyclInterface + function calls. + """ + _allowed_verbosity = ["warning", "error"] + if not verbosity in _allowed_verbosity: + raise ValueError( + f"Verbosity argument not understood. " + f"Permitted values are {_allowed_verbosity}" + ) + _init_logger(log_dir=log_dir) + _saved_verbosity = os.environ.get("DPCTL_VERBOSITY", None) + os.environ["DPCTL_VERBOSITY"] = verbosity + try: + yield + finally: + _shutdown_logger() + if _saved_verbosity: + os.environ["DPCTL_VERBOSITY"] = _saved_verbosity + else: + del os.environ["DPCTL_VERBOSITY"] diff --git a/dpctl/_host_task_util.hpp b/dpctl/_host_task_util.hpp new file mode 100644 index 0000000000..7871cdebfc --- /dev/null +++ b/dpctl/_host_task_util.hpp @@ -0,0 +1,74 @@ +//===--- _host_tasl_util.hpp - Implements async DECREF =// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements a utility function to schedule host task to a sycl +/// queue depending on given array of sycl events to decrement reference counts +/// for the given array of Python objects. +/// +/// N.B.: The host task attempts to acquire GIL, so queue wait, event wait and +/// other synchronization mechanisms should be called after releasing the GIL to +/// avoid deadlocks. +/// +//===----------------------------------------------------------------------===// + +#include "Python.h" +#include "syclinterface/dpctl_data_types.h" +#include + +int async_dec_ref(DPCTLSyclQueueRef QRef, + PyObject **obj_array, + size_t obj_array_size, + DPCTLSyclEventRef *ERefs, + size_t nERefs) +{ + + sycl::queue *q = reinterpret_cast(QRef); + + std::vector obj_vec; + obj_vec.reserve(obj_array_size); + for (size_t obj_id = 0; obj_id < obj_array_size; ++obj_id) { + obj_vec.push_back(obj_array[obj_id]); + } + + try { + q->submit([&](sycl::handler &cgh) { + for (size_t ev_id = 0; ev_id < nERefs; ++ev_id) { + cgh.depends_on( + *(reinterpret_cast(ERefs[ev_id]))); + } + cgh.host_task([obj_array_size, obj_vec]() { + // if the main thread has not finilized the interpreter yet + if (Py_IsInitialized()) { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + for (size_t i = 0; i < obj_array_size; ++i) { + Py_DECREF(obj_vec[i]); + } + PyGILState_Release(gstate); + } + }); + }); + } catch (const std::exception &e) { + return 1; + } + + return 0; +} diff --git a/dpctl/_sycl_context.pyx b/dpctl/_sycl_context.pyx index 404f27c99b..247a5d0f68 100644 --- a/dpctl/_sycl_context.pyx +++ b/dpctl/_sycl_context.pyx @@ -86,8 +86,9 @@ cdef class _SyclContext: cdef class SyclContext(_SyclContext): """ SyclContext(arg=None) - Python class representing ``cl::sycl::context``. There are multiple - ways to create a :class:`dpctl.SyclContext` object: + A Python wrapper for the :sycl_context:`sycl::context <>` C++ class. + + There are multiple ways to create a :class:`dpctl.SyclContext` object: - Invoking the constructor with no arguments creates a context using the default selector. @@ -479,9 +480,18 @@ cdef class SyclContext(_SyclContext): &_context_capsule_deleter ) -cdef api DPCTLSyclContextRef get_context_ref(SyclContext ctx): +cdef api DPCTLSyclContextRef SyclContext_GetContextRef(SyclContext ctx): """ C-API function to get opaque context reference from :class:`dpctl.SyclContext` instance. """ return ctx.get_context_ref() + + +cdef api SyclContext SyclContext_Make(DPCTLSyclContextRef CRef): + """ + C-API function to create :class:`dpctl.SyclContext` instance + from the given opaque context reference. + """ + cdef DPCTLSyclContextRef copied_CRef = DPCTLContext_Copy(CRef) + return SyclContext._create(copied_CRef) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 70302e9937..ee30002cce 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -175,13 +175,13 @@ cdef void _init_helper(_SyclDevice device, DPCTLSyclDeviceRef DRef): cdef class SyclDevice(_SyclDevice): """ SyclDevice(arg=None) - Python equivalent for cl::sycl::device class. + A Python wrapper for the :sycl_device:`sycl::device <>` C++ class. There are two ways of creating a SyclDevice instance: - by directly passing in a filter string to the class constructor. The filter string needs to conform to the - `DPC++ filter selector SYCL extension `_. + :oneapi_filter_selection:`DPC++ filter selector SYCL extension <>`. :Example: .. code-block:: python @@ -441,8 +441,8 @@ cdef class SyclDevice(_SyclDevice): return DPCTLDevice_HasAspect(self._device_ref, AT) @property - def has_aspect_usm_system_allocator(self): - cdef _aspect_type AT = _aspect_type._usm_system_allocator + def has_aspect_usm_system_allocations(self): + cdef _aspect_type AT = _aspect_type._usm_system_allocations return DPCTLDevice_HasAspect(self._device_ref, AT) @property @@ -1130,9 +1130,18 @@ cdef class SyclDevice(_SyclDevice): else: return str(relId) -cdef api DPCTLSyclDeviceRef get_device_ref(SyclDevice dev): +cdef api DPCTLSyclDeviceRef SyclDevice_GetDeviceRef(SyclDevice dev): """ C-API function to get opaque device reference from :class:`dpctl.SyclDevice` instance. """ return dev.get_device_ref() + + +cdef api SyclDevice SyclDevice_Make(DPCTLSyclDeviceRef DRef): + """ + C-API function to create :class:`dpctl.SyclDevice` instance + from the given opaque device reference. + """ + cdef DPCTLSyclDeviceRef copied_DRef = DPCTLDevice_Copy(DRef) + return SyclDevice._create(copied_DRef) diff --git a/dpctl/_sycl_event.pyx b/dpctl/_sycl_event.pyx index f78e19c326..c64478d3c6 100644 --- a/dpctl/_sycl_event.pyx +++ b/dpctl/_sycl_event.pyx @@ -57,13 +57,22 @@ __all__ = [ _logger = logging.getLogger(__name__) -cdef api DPCTLSyclEventRef get_event_ref(SyclEvent ev): +cdef api DPCTLSyclEventRef SyclEvent_GetEventRef(SyclEvent ev): """ C-API function to access opaque event reference from Python object of type :class:`dpctl.SyclEvent`. """ return ev.get_event_ref() +cdef api SyclEvent SyclEvent_Make(DPCTLSyclEventRef ERef): + """ + C-API function to create :class:`dpctl.SyclEvent` + instance from opaque sycl event reference. + """ + cdef DPCTLSyclEventRef copied_ERef = DPCTLEvent_Copy(ERef) + return SyclEvent._create(copied_ERef) + + cdef void _event_capsule_deleter(object o): cdef DPCTLSyclEventRef ERef = NULL if pycapsule.PyCapsule_IsValid(o, "SyclEventRef"): @@ -89,7 +98,6 @@ cdef class _SyclEvent: def __dealloc__(self): if (self._event_ref): - DPCTLEvent_Wait(self._event_ref) DPCTLEvent_Delete(self._event_ref) self._event_ref = NULL self.args = None @@ -215,7 +223,7 @@ cdef class SyclEvent(_SyclEvent): @staticmethod cdef void _wait(SyclEvent event): - DPCTLEvent_WaitAndThrow(event._event_ref) + with nogil: DPCTLEvent_WaitAndThrow(event._event_ref) @staticmethod def wait_for(event): @@ -250,13 +258,15 @@ cdef class SyclEvent(_SyclEvent): **SyclEventRef**. The ownership of the pointer inside the capsule is passed to the caller, and pointer is deleted when the capsule goes out of scope. + Returns: :class:`pycapsule`: A capsule object storing a copy of the - ``cl::sycl::event`` pointer belonging to thus + ``cl::sycl::event`` pointer belonging to a :class:`dpctl.SyclEvent` instance. Raises: ValueError: If the ``DPCTLEvent_Copy`` fails to copy the ``cl::sycl::event`` pointer. + """ cdef DPCTLSyclEventRef ERef = NULL ERef = DPCTLEvent_Copy(self._event_ref) @@ -359,4 +369,4 @@ cdef class SyclEvent(_SyclEvent): cpdef void wait(self): "Synchronously wait for completion of this event." - DPCTLEvent_Wait(self._event_ref) + with nogil: DPCTLEvent_Wait(self._event_ref) diff --git a/dpctl/_sycl_queue.pyx b/dpctl/_sycl_queue.pyx index 1091bbd765..30a64fec4c 100644 --- a/dpctl/_sycl_queue.pyx +++ b/dpctl/_sycl_queue.pyx @@ -65,11 +65,17 @@ import ctypes from .enum_types import backend_type from cpython cimport pycapsule +from cpython.ref cimport Py_DECREF, Py_INCREF, PyObject from libc.stdlib cimport free, malloc import collections.abc import logging + +cdef extern from "_host_task_util.hpp": + int async_dec_ref(DPCTLSyclQueueRef, PyObject **, size_t, DPCTLSyclEventRef *, size_t) nogil + + __all__ = [ "SyclQueue", "SyclKernelInvalidRangeError", @@ -714,12 +720,14 @@ cdef class SyclQueue(_SyclQueue): cdef _arg_data_type *kargty = NULL cdef DPCTLSyclEventRef *depEvents = NULL cdef DPCTLSyclEventRef Eref = NULL - cdef int ret + cdef int ret = 0 cdef size_t gRange[3] cdef size_t lRange[3] cdef size_t nGS = len(gS) cdef size_t nLS = len(lS) if lS is not None else 0 cdef size_t nDE = len(dEvents) if dEvents is not None else 0 + cdef PyObject **arg_objects = NULL + cdef ssize_t i = 0 # Allocate the arrays to be sent to DPCTLQueue_Submit kargs = malloc(len(args) * sizeof(void*)) @@ -820,11 +828,27 @@ cdef class SyclQueue(_SyclQueue): raise SyclKernelSubmitError( "Kernel submission to Sycl queue failed." ) + # increment reference counts to each argument + arg_objects = malloc(len(args) * sizeof(PyObject *)) + for i in range(len(args)): + arg_objects[i] = (args[i]) + Py_INCREF( arg_objects[i]) + + # schedule decrement + if async_dec_ref(self.get_queue_ref(), arg_objects, len(args), &Eref, 1): + # async task submission failed, decrement ref counts and wait + for i in range(len(args)): + arg_objects[i] = (args[i]) + Py_DECREF( arg_objects[i]) + with nogil: DPCTLEvent_Wait(Eref) - return SyclEvent._create(Eref, args) + # free memory + free(arg_objects) + + return SyclEvent._create(Eref, []) cpdef void wait(self): - DPCTLQueue_Wait(self._queue_ref) + with nogil: DPCTLQueue_Wait(self._queue_ref) cpdef memcpy(self, dest, src, size_t count): cdef void *c_dest @@ -846,7 +870,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.memcpy operation encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef prefetch(self, mem, size_t count=0): @@ -866,7 +890,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.prefetch encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef mem_advise(self, mem, size_t count, int advice): @@ -886,7 +910,7 @@ cdef class SyclQueue(_SyclQueue): raise RuntimeError( "SyclQueue.mem_advise operation encountered an error" ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) @property @@ -1001,9 +1025,18 @@ cdef class SyclQueue(_SyclQueue): self.sycl_device.print_device_info() -cdef api DPCTLSyclQueueRef get_queue_ref(SyclQueue q): +cdef api DPCTLSyclQueueRef SyclQueue_GetQueueRef(SyclQueue q): """ C-API function to get opaque queue reference from :class:`dpctl.SyclQueue` instance. """ return q.get_queue_ref() + + +cdef api SyclQueue SyclQueue_Make(DPCTLSyclQueueRef QRef): + """ + C-API function to create :class:`dpctl.SyclQueue` instance + from the given opaque queue reference. + """ + cdef DPCTLSyclQueueRef copied_QRef = DPCTLQueue_Copy(QRef) + return SyclQueue._create(copied_QRef) diff --git a/dpctl/_sycl_queue_manager.pyx b/dpctl/_sycl_queue_manager.pyx index c814b7286a..5f7ec50fad 100644 --- a/dpctl/_sycl_queue_manager.pyx +++ b/dpctl/_sycl_queue_manager.pyx @@ -229,7 +229,8 @@ def _help_numba_dppy(): @contextmanager def device_context(arg): """ - Yields a SYCL queue corresponding to the input filter string. + Yields a SYCL queue corresponding to the input queue object, device object, + or device filter selector string. This context manager "activates", *i.e.*, sets as the currently usable queue, the SYCL queue defined by the argument `arg`. @@ -242,25 +243,25 @@ def device_context(arg): This context manager uses context factories to create and activate nested contexts. Args: - - queue_str (str) : A string corresponding to the DPC++ filter selector. + arg : A :class:`dpctl.SyclQueue` object, or a :class:`dpctl.SyclDevice` + object, or a filter selector string. Yields: - :class:`.SyclQueue`: A SYCL queue corresponding to the specified - filter string. + :class:`dpctl.SyclQueue`: A SYCL queue corresponding to the specified + input device, queue, or filter string. Raises: SyclQueueCreationError: If the SYCL queue creation failed. :Example: - To create a scope within which the Level Zero GPU number 0 is active, - a programmer needs to do the following. + The following example sets current queue targeting specific device + indicated with filter selector string in the scope of `with` block: .. code-block:: python import dpctl with dpctl.device_context("level0:gpu:0"): - pass + do_something_on_gpu0() The following example registers nested context factory: diff --git a/dpctl/_version.py b/dpctl/_version.py index 395c950102..d166fbc8b9 100644 --- a/dpctl/_version.py +++ b/dpctl/_version.py @@ -1,3 +1,4 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -5,7 +6,7 @@ # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer) """Git implementation of _version.py.""" @@ -14,6 +15,7 @@ import re import subprocess import sys +from typing import Callable, Dict def get_keywords(): @@ -41,8 +43,8 @@ def get_config(): cfg.VCS = "git" cfg.style = "" cfg.tag_prefix = "" - cfg.parentdir_prefix = "DP-Glue-" - cfg.versionfile_source = "dpglue/_version.py" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "dpctl/_version.py" cfg.verbose = False return cfg @@ -51,40 +53,36 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f - return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -96,15 +94,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): @@ -116,25 +112,18 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -147,22 +136,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -170,10 +158,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -186,11 +178,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +191,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -207,30 +199,28 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] + r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -238,10 +228,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"\*" - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -249,24 +242,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%s%s" % (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -276,6 +261,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -284,16 +302,17 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] + git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + # unparsable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) return pieces # tag @@ -302,12 +321,10 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] + pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -318,13 +335,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -355,25 +373,74 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered @@ -404,12 +471,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -469,23 +565,25 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -495,13 +593,9 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): @@ -515,7 +609,8 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) except NotThisMethod: pass @@ -524,16 +619,13 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -547,10 +639,6 @@ def get_versions(): except NotThisMethod: pass - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/dpctl/apis/include/dpctl4pybind11.hpp b/dpctl/apis/include/dpctl4pybind11.hpp new file mode 100644 index 0000000000..80d01af721 --- /dev/null +++ b/dpctl/apis/include/dpctl4pybind11.hpp @@ -0,0 +1,176 @@ +//===----------- dpctl4pybind11.h - Headers for type pybind11 casters -*-C-*- +//===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines imports for dcptl's Python C-API +//===----------------------------------------------------------------------===// + +#pragma once + +#include "dpctl_capi.h" +#include +#include + +namespace py = pybind11; + +namespace pybind11 +{ +namespace detail +{ + +/* This type caster associates ``sycl::queue`` C++ class with + * :class:`dpctl.SyclQueue` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::queue, _("dpctl.SyclQueue")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclQueueType)) { + DPCTLSyclQueueRef QRef = SyclQueue_GetQueueRef( + reinterpret_cast(source)); + sycl::queue *q = reinterpret_cast(QRef); + value = *q; + return true; + } + else if (source == Py_None) { + value = sycl::queue{}; + return true; + } + else { + throw py::type_error( + "Input is of unexpected type, expected dpctl.SyclQueue"); + } + } + + static handle cast(sycl::queue src, return_value_policy, handle) + { + auto tmp = SyclQueue_Make(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; + +/* This type caster associates ``sycl::device`` C++ class with + * :class:`dpctl.SyclDevice` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::device, _("dpctl.SyclDevice")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclDeviceType)) { + DPCTLSyclDeviceRef DRef = SyclDevice_GetDeviceRef( + reinterpret_cast(source)); + sycl::device *d = reinterpret_cast(DRef); + value = *d; + return true; + } + else if (source == Py_None) { + value = sycl::device{}; + return true; + } + else { + throw py::type_error( + "Input is of unexpected type, expected dpctl.SyclDevice"); + } + } + + static handle cast(sycl::device src, return_value_policy, handle) + { + auto tmp = SyclDevice_Make(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; + +/* This type caster associates ``sycl::context`` C++ class with + * :class:`dpctl.SyclContext` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::context, _("dpctl.SyclContext")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclContextType)) { + DPCTLSyclContextRef CRef = SyclContext_GetContextRef( + reinterpret_cast(source)); + sycl::context *ctx = reinterpret_cast(CRef); + value = *ctx; + return true; + } + else { + throw py::type_error( + "Input is of unexpected type, expected dpctl.SyclContext"); + } + } + + static handle cast(sycl::context src, return_value_policy, handle) + { + auto tmp = + SyclContext_Make(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; + +/* This type caster associates ``sycl::event`` C++ class with + * :class:`dpctl.SyclEvent` for the purposes of generation of + * Python bindings by pybind11. + */ +template <> struct type_caster +{ +public: + PYBIND11_TYPE_CASTER(sycl::event, _("dpctl.SyclEvent")); + + bool load(handle src, bool) + { + PyObject *source = src.ptr(); + if (PyObject_TypeCheck(source, &PySyclEventType)) { + DPCTLSyclEventRef ERef = SyclEvent_GetEventRef( + reinterpret_cast(source)); + sycl::event *ev = reinterpret_cast(ERef); + value = *ev; + return true; + } + else { + throw py::type_error( + "Input is of unexpected type, expected dpctl.SyclEvent"); + } + } + + static handle cast(sycl::event src, return_value_policy, handle) + { + auto tmp = SyclEvent_Make(reinterpret_cast(&src)); + return handle(reinterpret_cast(tmp)); + } +}; +} // namespace detail +} // namespace pybind11 diff --git a/dpctl/apis/include/dpctl_capi.h b/dpctl/apis/include/dpctl_capi.h new file mode 100644 index 0000000000..18b01f9e19 --- /dev/null +++ b/dpctl/apis/include/dpctl_capi.h @@ -0,0 +1,60 @@ +//===----------- dpctl_capi.h - Headers for dpctl's C-API -*-C-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines imports for dcptl's Python C-API +//===----------------------------------------------------------------------===// + +#pragma once + +// clang-format off +// Ordering of includes is important here. dpctl_sycl_types defines types +// used by dpctl's Python C-API headers. +#include "syclinterface/dpctl_sycl_types.h" +#include "../_sycl_device.h" +#include "../_sycl_device_api.h" +#include "../_sycl_context.h" +#include "../_sycl_context_api.h" +#include "../_sycl_event.h" +#include "../_sycl_event_api.h" +#include "../_sycl_queue.h" +#include "../_sycl_queue_api.h" +#include "../memory/_memory.h" +#include "../memory/_memory_api.h" +#include "../tensor/_usmarray.h" +#include "../tensor/_usmarray_api.h" +// clang-format on + +/* + * Function to import dpctl and make C-API functions available. + * C functions can use dpctl's C-API functions without linking to + * shared objects defining this symbols, if they call `import_dpctl()` + * prior to using those symbols. + */ +void import_dpctl(void) +{ + import_dpctl___sycl_device(); + import_dpctl___sycl_context(); + import_dpctl___sycl_event(); + import_dpctl___sycl_queue(); + import_dpctl__memory___memory(); + import_dpctl__tensor___usmarray(); + return; +} diff --git a/dpctl/apis/include/dpctl_sycl_interface.h b/dpctl/apis/include/dpctl_sycl_interface.h new file mode 100644 index 0000000000..4f4f4c2b19 --- /dev/null +++ b/dpctl/apis/include/dpctl_sycl_interface.h @@ -0,0 +1,43 @@ +//=== syclinterace.h - single include header for libsyclinterface -*-C-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file includes all the headers of syclinterface/ +//===----------------------------------------------------------------------===// + +#pragma once + +// clang-format off +#include "syclinterface/dpctl_sycl_types.h" +#include "syclinterface/dpctl_sycl_enum_types.h" +#include "syclinterface/dpctl_service.h" +#include "syclinterface/dpctl_vector.h" +#include "syclinterface/dpctl_utils.h" +#include "syclinterface/dpctl_sycl_device_selector_interface.h" +#include "syclinterface/dpctl_sycl_context_interface.h" +#include "syclinterface/dpctl_sycl_device_interface.h" +#include "syclinterface/dpctl_sycl_event_interface.h" +#include "syclinterface/dpctl_sycl_platform_interface.h" +#include "syclinterface/dpctl_sycl_queue_interface.h" +#include "syclinterface/dpctl_sycl_usm_interface.h" +#include "syclinterface/dpctl_sycl_device_manager.h" +#include "syclinterface/dpctl_sycl_platform_manager.h" +#include "syclinterface/dpctl_sycl_queue_manager.h" +// clang-format on diff --git a/dpctl/cmake/copy_existing.cmake b/dpctl/cmake/copy_existing.cmake new file mode 100644 index 0000000000..242dc29257 --- /dev/null +++ b/dpctl/cmake/copy_existing.cmake @@ -0,0 +1,3 @@ +if (EXISTS ${SOURCE_FILE}) + configure_file(${SOURCE_FILE} ${DEST} COPYONLY) +endif() diff --git a/dpctl/enum_types.py b/dpctl/enum_types.py index 5add734167..bdf95959c0 100644 --- a/dpctl/enum_types.py +++ b/dpctl/enum_types.py @@ -33,14 +33,19 @@ class device_type(Enum): """ An enumeration of supported SYCL device types. - ================== ============ - Device type Enum value - ================== ============ - gpu 1 - cpu 2 - accelerator 3 - host 4 - ================== ============ + :Example: + .. code-block:: python + + import dpctl + + # filter GPU devices amongst available SYCL devices + gpu_devs = [ + d for d in dpctl.get_devices() if ( + d.device_type == dpctl.device_type.gpu + ) ] + + # alternatively, get GPU devices directly + gpu_devs2 = dpctl.get_devices(device_type=dpctl.device_type.gpu) """ all = auto() @@ -56,15 +61,15 @@ class backend_type(Enum): """ An enumeration of supported SYCL backends. - ================== ============ - Name of backend Enum value - ================== ============ - opencl 1 - level_zero 2 - cuda 3 - host 4 - ================== ============ + :Example: + .. code-block:: python + + import dpctl + # create a SYCL device with OpenCL backend using filter selector + d = dpctl.SyclDevice("opencl") + print(d.backend) + # Possible output: """ all = auto() @@ -75,6 +80,17 @@ class backend_type(Enum): class event_status_type(Enum): + """ + An enumeration of SYCL event states. + + :Example: + .. code-block:: python + + import dpctl + ev = dpctl.SyclEvent() + print(ev.execution_status ) + # Possible output: + """ unknown_status = auto() submitted = auto() diff --git a/dpctl/memory/CMakeLists.txt b/dpctl/memory/CMakeLists.txt new file mode 100644 index 0000000000..c4f6d8469f --- /dev/null +++ b/dpctl/memory/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/memory") +endforeach() diff --git a/dpctl/memory/__init__.py b/dpctl/memory/__init__.py index 7b4032f1d7..593f03696e 100644 --- a/dpctl/memory/__init__.py +++ b/dpctl/memory/__init__.py @@ -15,10 +15,8 @@ # limitations under the License. """ - **Data Parallel Control Memory** - - `dpctl.memory` provides Python objects for untyped USM memory - container of bytes for each kind of USM pointers: shared pointers, + **Data Parallel Control Memory** provides Python objects for untyped USM + memory container of bytes for each kind of USM pointers: shared pointers, device pointers and host pointers. Shared and host pointers are accessible from both host and a device, diff --git a/dpctl/memory/_memory.pyx b/dpctl/memory/_memory.pyx index 6a98cf9857..c40cd0fb89 100644 --- a/dpctl/memory/_memory.pyx +++ b/dpctl/memory/_memory.pyx @@ -94,7 +94,7 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, src_ptr, nbytes ) - DPCTLEvent_Wait(E1Ref) + with nogil: DPCTLEvent_Wait(E1Ref) E2Ref = DPCTLQueue_Memcpy( dest_queue.get_queue_ref(), @@ -102,7 +102,7 @@ cdef void copy_via_host(void *dest_ptr, SyclQueue dest_queue, &host_buf[0], nbytes ) - DPCTLEvent_Wait(E2Ref) + with nogil: DPCTLEvent_Wait(E2Ref) DPCTLEvent_Delete(E1Ref) DPCTLEvent_Delete(E2Ref) @@ -142,6 +142,7 @@ cdef class _Memory: cdef _cinit_alloc(self, Py_ssize_t alignment, Py_ssize_t nbytes, bytes ptr_type, SyclQueue queue): cdef DPCTLSyclUSMRef p = NULL + cdef DPCTLSyclQueueRef QRef = NULL self._cinit_empty() @@ -149,27 +150,28 @@ cdef class _Memory: if queue is None: queue = dpctl.SyclQueue() + QRef = queue.get_queue_ref() if (ptr_type == b"shared"): if alignment > 0: - p = DPCTLaligned_alloc_shared( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_shared( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_shared(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_shared(nbytes, QRef) elif (ptr_type == b"host"): if alignment > 0: - p = DPCTLaligned_alloc_host( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_host( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_host(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_host(nbytes, QRef) elif (ptr_type == b"device"): if (alignment > 0): - p = DPCTLaligned_alloc_device( - alignment, nbytes, queue.get_queue_ref() + with nogil: p = DPCTLaligned_alloc_device( + alignment, nbytes, QRef ) else: - p = DPCTLmalloc_device(nbytes, queue.get_queue_ref()) + with nogil: p = DPCTLmalloc_device(nbytes, QRef) else: raise RuntimeError( "Pointer type is unknown: {}".format( @@ -295,6 +297,15 @@ cdef class _Memory: def __get__(self): return self.queue.get_sycl_device() + property sycl_queue: + """ + :class:`dpctl.SyclQueue` with :class:`dpctl.SyclContext` the + USM allocation is bound to and :class:`dpctl.SyclDevice` it was + allocated on. + """ + def __get__(self): + return self.queue + def __repr__(self): return ( "" @@ -389,7 +400,7 @@ cdef class _Memory: self.memory_ptr, # source self.nbytes ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) return obj @@ -414,7 +425,7 @@ cdef class _Memory: &host_buf[0], # source buf_len ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) cpdef copy_from_device(self, object sycl_usm_ary): @@ -456,7 +467,7 @@ cdef class _Memory: src_buf.p, src_buf.nbytes ) - DPCTLEvent_Wait(ERef) + with nogil: DPCTLEvent_Wait(ERef) DPCTLEvent_Delete(ERef) else: copy_via_host( @@ -740,11 +751,30 @@ def as_usm_memory(obj): ) -cdef api DPCTLSyclUSMRef get_usm_pointer(_Memory obj): +cdef api DPCTLSyclUSMRef Memory_GetUsmPointer(_Memory obj): + "Pointer of USM allocation" return obj.memory_ptr -cdef api DPCTLSyclContextRef get_context(_Memory obj): +cdef api DPCTLSyclContextRef Memory_GetContextRef(_Memory obj): + "Context reference to which USM allocation is bound" return obj.queue._context.get_context_ref() -cdef api size_t get_nbytes(_Memory obj): +cdef api DPCTLSyclQueueRef Memory_GetQueueRef(_Memory obj): + """Queue associated with this allocation, used + for copying, population, etc.""" + return obj.queue.get_queue_ref() + +cdef api size_t Memory_GetNumBytes(_Memory obj): + "Size of the allocation in bytes." return obj.nbytes + +cdef api object Memory_Make( + DPCTLSyclUSMRef ptr, + size_t nbytes, + DPCTLSyclQueueRef QRef, + object owner +): + "Create _Memory Python object from preallocated memory." + return _Memory.create_from_usm_pointer_size_qref( + ptr, nbytes, QRef, memory_owner=owner + ) diff --git a/dpctl/program/CMakeLists.txt b/dpctl/program/CMakeLists.txt new file mode 100644 index 0000000000..f10706ed9d --- /dev/null +++ b/dpctl/program/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/program") +endforeach() diff --git a/dpctl/program/__init__.py b/dpctl/program/__init__.py index 58a46bb2eb..1d72e90e72 100644 --- a/dpctl/program/__init__.py +++ b/dpctl/program/__init__.py @@ -15,10 +15,9 @@ # limitations under the License. """ - **Data Parallel Control Program** - - `dpctl.program` module provides a way to create a SYCL kernel from either a - source string or SPIR-V binary file. + **Data Parallel Control Program** provides a way to create a SYCL kernel + from either an OpenCL program represented as a string or a SPIR-V binary + file. """ from ._program import ( diff --git a/dpctl/tensor/CMakeLists.txt b/dpctl/tensor/CMakeLists.txt new file mode 100644 index 0000000000..855bba7e8a --- /dev/null +++ b/dpctl/tensor/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/tensor") + target_include_directories(${_trgt} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) +endforeach() diff --git a/dpctl/tensor/__init__.py b/dpctl/tensor/__init__.py index 32e1e1ce46..7aee5ebad1 100644 --- a/dpctl/tensor/__init__.py +++ b/dpctl/tensor/__init__.py @@ -15,33 +15,29 @@ # limitations under the License. """ - **Data Parallel Tensor Collection** - - ``dpctl.tensor`` is an experimental collection of tensor implementations - that will implement future Python data API - (https://data-apis.github.io/array-api/latest/). - - Available tensor implementations: - - ``numpy_usm_shared``: - Provides a ``numpy.ndarray`` sub-class whose underlying memory buffer - is allocated with a USM shared memory allocator. + **Data Parallel Tensor Collection** is a collection of tensor + implementations that implement Python data API + (https://data-apis.github.io/array-api/latest/) standard. """ -from dpctl.tensor._copy_utils import astype, copy -from dpctl.tensor._copy_utils import copy_from_numpy as from_numpy -from dpctl.tensor._copy_utils import copy_to_numpy as asnumpy -from dpctl.tensor._copy_utils import copy_to_numpy as to_numpy +from dpctl.tensor._copy_utils import asnumpy, astype, copy, from_numpy, to_numpy +from dpctl.tensor._ctors import asarray, empty +from dpctl.tensor._device import Device +from dpctl.tensor._dlpack import from_dlpack from dpctl.tensor._reshape import reshape from dpctl.tensor._usmarray import usm_ndarray __all__ = [ + "Device", "usm_ndarray", + "asarray", "astype", "copy", + "empty", "reshape", "from_numpy", "to_numpy", "asnumpy", + "from_dlpack", ] diff --git a/dpctl/tensor/_copy_utils.py b/dpctl/tensor/_copy_utils.py index 7a7ba918f6..cb3d434689 100644 --- a/dpctl/tensor/_copy_utils.py +++ b/dpctl/tensor/_copy_utils.py @@ -19,6 +19,7 @@ import dpctl.memory as dpm import dpctl.tensor as dpt +from dpctl.tensor._device import normalize_queue_device def contract_iter2(shape, strides1, strides2): @@ -64,20 +65,26 @@ def contract_iter2(shape, strides1, strides2): return (sh, st1, disp1, st2, disp2) -def has_memory_overlap(x1, x2): - m1 = dpm.as_usm_memory(x1) - m2 = dpm.as_usm_memory(x2) - if m1.sycl_device == m2.sycl_device: - p1_beg = m1._pointer - p1_end = p1_beg + m1.nbytes - p2_beg = m2._pointer - p2_end = p2_beg + m2.nbytes - return p1_beg > p2_end or p2_beg < p1_end +def _has_memory_overlap(x1, x2): + if x1.size and x2.size: + m1 = dpm.as_usm_memory(x1) + m2 = dpm.as_usm_memory(x2) + # can only overlap if bound to the same context + if m1.sycl_context == m2.sycl_context: + p1_beg = m1._pointer + p1_end = p1_beg + m1.nbytes + p2_beg = m2._pointer + p2_end = p2_beg + m2.nbytes + # may intersect if not ((p1_beg >= p2_end) or (p2_beg >= p2_end)) + return (p1_beg < p2_end) and (p2_beg < p1_end) + else: + return False else: + # zero element array do not overlap anything return False -def copy_to_numpy(ary): +def _copy_to_numpy(ary): if type(ary) is not dpt.usm_ndarray: raise TypeError h = ary.usm_data.copy_to_host().view(ary.dtype) @@ -93,12 +100,12 @@ def copy_to_numpy(ary): ) -def copy_from_numpy(np_ary, usm_type="device", queue=None): +def _copy_from_numpy(np_ary, usm_type="device", sycl_queue=None): "Copies numpy array `np_ary` into a new usm_ndarray" # This may peform a copy to meet stated requirements Xnp = np.require(np_ary, requirements=["A", "O", "C", "E"]) - if queue: - ctor_kwargs = {"queue": queue} + if sycl_queue: + ctor_kwargs = {"queue": sycl_queue} else: ctor_kwargs = dict() Xusm = dpt.usm_ndarray( @@ -111,10 +118,17 @@ def copy_from_numpy(np_ary, usm_type="device", queue=None): return Xusm -def copy_from_numpy_into(dst, np_ary): +def _copy_from_numpy_into(dst, np_ary): + "Copies `np_ary` into `dst` of type :class:`dpctl.tensor.usm_ndarray" if not isinstance(np_ary, np.ndarray): raise TypeError("Expected numpy.ndarray, got {}".format(type(np_ary))) src_ary = np.broadcast_to(np.asarray(np_ary, dtype=dst.dtype), dst.shape) + if src_ary.size and (dst.flags & 1) and src_ary.flags["C"]: + dpm.as_usm_memory(dst).copy_from_host(src_ary.reshape((-1,)).view("u1")) + return + if src_ary.size and (dst.flags & 2) and src_ary.flags["F"]: + dpm.as_usm_memory(dst).copy_from_host(src_ary.reshape((-1,)).view("u1")) + return for i in range(dst.size): mi = np.unravel_index(i, dst.shape) host_buf = np.array(src_ary[mi], ndmin=1).view("u1") @@ -122,6 +136,54 @@ def copy_from_numpy_into(dst, np_ary): usm_mem.copy_from_host(host_buf) +def from_numpy(np_ary, device=None, usm_type="device", sycl_queue=None): + """ + from_numpy(arg, device=None, usm_type="device", sycl_queue=None) + + Creates :class:`dpctl.tensor.usm_ndarray` from instance of + `numpy.ndarray`. + + Args: + arg: An instance of `numpy.ndarray` + device: array API specification of device where the output array + is created. + sycl_queue: a :class:`dpctl.SyclQueue` used to create the output + array is created + """ + q = normalize_queue_device(sycl_queue=sycl_queue, device=device) + return _copy_from_numpy(np_ary, usm_type=usm_type, sycl_queue=q) + + +def to_numpy(usm_ary): + """ + to_numpy(usm_ary) + + Copies content of :class:`dpctl.tensor.usm_ndarray` instance `usm_ary` + into `numpy.ndarray` instance of the same shape and same data type. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + An instance of `numpy.ndarray` populated with content of `usm_ary`. + """ + return _copy_to_numpy(usm_ary) + + +def asnumpy(usm_ary): + """ + asnumpy(usm_ary) + + Copies content of :class:`dpctl.tensor.usm_ndarray` instance `usm_ary` + into `numpy.ndarray` instance of the same shape and same data type. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + An instance of `numpy.ndarray` populated with content of `usm_ary`. + """ + return _copy_to_numpy(usm_ary) + + class Dummy: def __init__(self, iface): self.__sycl_usm_array_interface__ = iface @@ -137,10 +199,13 @@ def copy_same_dtype(dst, src): if dst.dtype != src.dtype: raise ValueError + if dst.size == 0: + return + # check that memory regions do not overlap - if has_memory_overlap(dst, src): - tmp = copy_to_numpy(src) - copy_from_numpy_into(dst, tmp) + if _has_memory_overlap(dst, src): + tmp = _copy_to_numpy(src) + _copy_from_numpy_into(dst, tmp) return if (dst.flags & 1) and (src.flags & 1): @@ -184,10 +249,10 @@ def copy_same_shape(dst, src): return # check that memory regions do not overlap - if has_memory_overlap(dst, src): - tmp = copy_to_numpy(src) + if _has_memory_overlap(dst, src): + tmp = _copy_to_numpy(src) tmp = tmp.astype(dst.dtype) - copy_from_numpy_into(dst, tmp) + _copy_from_numpy_into(dst, tmp) return # simplify strides @@ -218,7 +283,7 @@ def copy_same_shape(dst, src): mdst.copy_from_host(tmp.view("u1")) -def copy_from_usm_ndarray_to_usm_ndarray(dst, src): +def _copy_from_usm_ndarray_to_usm_ndarray(dst, src): if type(dst) is not dpt.usm_ndarray or type(src) is not dpt.usm_ndarray: raise TypeError @@ -260,12 +325,11 @@ def copy(usm_ary, order="K"): following NumPy's conventions. The `order` keywords can be one of the following: - "C": C-contiguous memory layout - "F": Fotrant-contiguous memory layout - "A": Fotrant-contiguous if the input array is - F-contiguous, and C-contiguous otherwise - "K": match the layout of `usm_ary` as closely - as possible. + - "C": C-contiguous memory layout + - "F": Fortran-contiguous memory layout + - "A": Fortran-contiguous if the input array is also Fortran-contiguous, + otherwise C-contiguous + - "K": match the layout of `usm_ary` as closely as possible. """ if not isinstance(usm_ary, dpt.usm_ndarray): @@ -389,7 +453,7 @@ def astype(usm_ary, newdtype, order="K", casting="unsafe", copy=True): buffer=R.usm_data, strides=new_strides, ) - copy_from_usm_ndarray_to_usm_ndarray(R, usm_ary) + _copy_from_usm_ndarray_to_usm_ndarray(R, usm_ary) return R else: return usm_ary diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py new file mode 100644 index 0000000000..f080537682 --- /dev/null +++ b/dpctl/tensor/_ctors.py @@ -0,0 +1,446 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +import dpctl +import dpctl.memory as dpm +import dpctl.tensor as dpt +import dpctl.utils +from dpctl.tensor._device import normalize_queue_device + +_empty_tuple = tuple() +_host_set = frozenset([None]) + + +def _array_info_dispatch(obj): + if isinstance(obj, dpt.usm_ndarray): + return obj.shape, obj.dtype, frozenset([obj.sycl_queue]) + elif isinstance(obj, np.ndarray): + return obj.shape, obj.dtype, _host_set + elif isinstance(obj, range): + return (len(obj),), int, _host_set + elif isinstance(obj, bool): + return _empty_tuple, bool, _host_set + elif isinstance(obj, float): + return _empty_tuple, float, _host_set + elif isinstance(obj, int): + return _empty_tuple, int, _host_set + elif isinstance(obj, complex): + return _empty_tuple, complex, _host_set + elif isinstance(obj, (list, tuple, range)): + return _array_info_sequence(obj) + else: + raise ValueError(type(obj)) + + +def _array_info_sequence(li): + assert isinstance(li, (list, tuple, range)) + n = len(li) + dim = None + dt = None + device = frozenset() + for el in li: + el_dim, el_dt, el_dev = _array_info_dispatch(el) + if dim is None: + dim = el_dim + dt = np.promote_types(el_dt, el_dt) + device = device.union(el_dev) + elif el_dim == dim: + dt = np.promote_types(dt, el_dt) + device = device.union(el_dev) + else: + raise ValueError( + "Inconsistent dimensions, {} and {}".format(dim, el_dim) + ) + if dim is None: + dim = tuple() + dt = float + device = _host_set + return (n,) + dim, dt, device + + +def _asarray_from_usm_ndarray( + usm_ndary, + dtype=None, + copy=None, + usm_type=None, + sycl_queue=None, + order="K", +): + if not isinstance(usm_ndary, dpt.usm_ndarray): + raise TypeError( + f"Expected dpctl.tensor.usm_ndarray, got {type(usm_ndary)}" + ) + if dtype is None: + dtype = usm_ndary.dtype + if usm_type is None: + usm_type = usm_ndary.usm_type + if sycl_queue is not None: + exec_q = dpctl.utils.get_execution_queue( + [usm_ndary.sycl_queue, sycl_queue] + ) + copy_q = normalize_queue_device(sycl_queue=sycl_queue, device=exec_q) + else: + copy_q = usm_ndary.sycl_queue + # Conditions for zero copy: + can_zero_copy = copy is not True + # dtype is unchanged + can_zero_copy = can_zero_copy and dtype == usm_ndary.dtype + # USM allocation type is unchanged + can_zero_copy = can_zero_copy and usm_type == usm_ndary.usm_type + # sycl_queue is unchanged + can_zero_copy = can_zero_copy and copy_q is usm_ndary.sycl_queue + # order is unchanged + c_contig = usm_ndary.flags & 1 + f_contig = usm_ndary.flags & 2 + fc_contig = usm_ndary.flags & 3 + if can_zero_copy: + if order == "C" and c_contig: + pass + elif order == "F" and f_contig: + pass + elif order == "A" and fc_contig: + pass + elif order == "K": + pass + else: + can_zero_copy = False + if copy is False and can_zero_copy is False: + raise ValueError("asarray(..., copy=False) is not possible") + if can_zero_copy: + return usm_ndary + if order == "A": + order = "F" if f_contig and not c_contig else "C" + if order == "K" and fc_contig: + order = "C" if c_contig else "F" + if order == "K": + # new USM allocation + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=dtype, + buffer=usm_type, + order="C", + buffer_ctor_kwargs={"queue": copy_q}, + ) + original_strides = usm_ndary.strides + ind = sorted( + range(usm_ndary.ndim), + key=lambda i: abs(original_strides[i]), + reverse=True, + ) + new_strides = tuple(res.strides[ind[i]] for i in ind) + # reuse previously made USM allocation + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=res.dtype, + buffer=res.usm_data, + strides=new_strides, + ) + else: + res = dpt.usm_ndarray( + usm_ndary.shape, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": copy_q}, + ) + # FIXME: call copy_to when implemented + res[(slice(None, None, None),) * res.ndim] = usm_ndary + return res + + +def _asarray_from_numpy_ndarray( + ary, dtype=None, usm_type=None, sycl_queue=None, order="K" +): + if not isinstance(ary, np.ndarray): + raise TypeError(f"Expected numpy.ndarray, got {type(ary)}") + if usm_type is None: + usm_type = "device" + if dtype is None: + dtype = ary.dtype + copy_q = normalize_queue_device(sycl_queue=None, device=sycl_queue) + f_contig = ary.flags["F"] + c_contig = ary.flags["C"] + fc_contig = f_contig or c_contig + if order == "A": + order = "F" if f_contig and not c_contig else "C" + if order == "K" and fc_contig: + order = "C" if c_contig else "F" + if order == "K": + # new USM allocation + res = dpt.usm_ndarray( + ary.shape, + dtype=dtype, + buffer=usm_type, + order="C", + buffer_ctor_kwargs={"queue": copy_q}, + ) + original_strides = ary.strides + ind = sorted( + range(ary.ndim), + key=lambda i: abs(original_strides[i]), + reverse=True, + ) + new_strides = tuple(res.strides[ind[i]] for i in ind) + # reuse previously made USM allocation + res = dpt.usm_ndarray( + res.shape, dtype=res.dtype, buffer=res.usm_data, strides=new_strides + ) + else: + res = dpt.usm_ndarray( + ary.shape, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": copy_q}, + ) + # FIXME: call copy_to when implemented + res[(slice(None, None, None),) * res.ndim] = ary + return res + + +def _is_object_with_buffer_protocol(obj): + "Returns True if object support Python buffer protocol" + try: + # use context manager to ensure + # buffer is instantly released + with memoryview(obj): + return True + except TypeError: + return False + + +def asarray( + obj, + dtype=None, + device=None, + copy=None, + usm_type=None, + sycl_queue=None, + order="K", +): + """ + Converts `obj` to :class:`dpctl.tensor.usm_ndarray`. + + Args: + obj: Python object to convert. Can be an instance of `usm_ndarray`, + an object representing SYCL USM allocation and implementing + `__sycl_usm_array_interface__` protocol, an instance + of `numpy.ndarray`, an object supporting Python buffer protocol, + a Python scalar, or a (possibly nested) sequence of Python scalars. + dtype (data type, optional): output array data type. If `dtype` is + `None`, the output array data type is inferred from data types in + `obj`. Default: `None` + copy (`bool`, optional): boolean indicating whether or not to copy the + input. If `True`, always creates a copy. If `False`, need to copy + raises `ValueError`. If `None`, try to reuse existing memory + allocations if possible, but allowed to perform a copy otherwise. + Default: `None`. + order ("C","F","A","K", optional): memory layout of the output array. + Default: "C" + device (optional): array API concept of device where the output array + is created. `device` can be `None`, a oneAPI filter selector string, + an instance of :class:`dpctl.SyclDevice` corresponding to a + non-partitioned SYCL device, an instance of + :class:`dpctl.SyclQueue`, or a `Device` object returnedby + `dpctl.tensor.usm_array.device`. Default: `None`. + usm_type ("device"|"shared"|"host", optional): The type of SYCL USM + allocation for the output array. For `usm_type=None` the allocation + type is inferred from the input if `obj` has USM allocation, or + `"device"` is used instead. Default: `None`. + sycl_queue: (:class:`dpctl.SyclQueue`, optional): The SYCL queue to use + for output array allocation and copying. `sycl_queue` and `device` + are exclusive keywords, i.e. use one or another. If both are + specified, a `TypeError` is raised unless both imply the same + underlying SYCL queue to be used. If both a `None`, the + `dpctl.SyclQueue()` is used for allocation and copying. + Default: `None`. + """ + # 1. Check that copy is a valid keyword + if copy not in [None, True, False]: + raise TypeError( + "Recognized copy keyword values should be True, False, or None" + ) + # 2. Check that dtype is None, or a valid dtype + if dtype is not None: + dtype = np.dtype(dtype) + # 3. Validate order + if not isinstance(order, str): + raise TypeError( + f"Expected order keyword to be of type str, got {type(order)}" + ) + if len(order) == 0 or order[0] not in "KkAaCcFf": + raise ValueError( + "Unrecognized order keyword value, expecting 'K', 'A', 'F', or 'C'." + ) + else: + order = order[0].upper() + # 4. Check that usm_type is None, or a valid value + if usm_type is not None: + if isinstance(usm_type, str): + if usm_type not in ["device", "shared", "host"]: + raise ValueError( + f"Unrecognized value of usm_type={usm_type}, " + "expected 'device', 'shared', 'host', or None." + ) + else: + raise TypeError( + f"Expected usm_type to be a str or None, got {type(usm_type)}" + ) + # 5. Normalize device/sycl_queue [keep it None if was None] + if device is not None or sycl_queue is not None: + sycl_queue = normalize_queue_device( + sycl_queue=sycl_queue, device=device + ) + + # handle instance(obj, usm_ndarray) + if isinstance(obj, dpt.usm_ndarray): + return _asarray_from_usm_ndarray( + obj, + dtype=dtype, + copy=copy, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif hasattr(obj, "__sycl_usm_array_interface__"): + sua_iface = getattr(obj, "__sycl_usm_array_interface__") + membuf = dpm.as_usm_memory(obj) + ary = dpt.usm_ndarray( + sua_iface["shape"], + dtype=sua_iface["typestr"], + buffer=membuf, + strides=sua_iface.get("strides", None), + ) + return _asarray_from_usm_ndarray( + ary, + dtype=dtype, + copy=copy, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif isinstance(obj, np.ndarray): + if copy is False: + raise ValueError( + "Converting numpy.ndarray to usm_ndarray requires a copy" + ) + return _asarray_from_numpy_ndarray( + obj, + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif _is_object_with_buffer_protocol(obj): + if copy is False: + raise ValueError( + f"Converting {type(obj)} to usm_ndarray requires a copy" + ) + return _asarray_from_numpy_ndarray( + np.array(obj), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + elif isinstance(obj, (list, tuple, range)): + if copy is False: + raise ValueError( + "Converting Python sequence to usm_ndarray requires a copy" + ) + _, dt, devs = _array_info_sequence(obj) + if devs == _host_set: + return _asarray_from_numpy_ndarray( + np.asarray(obj, dtype=dtype, order=order), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order=order, + ) + # for sequences + raise NotImplementedError( + "Converting Python sequences is not implemented" + ) + if copy is False: + raise ValueError( + f"Converting {type(obj)} to usm_ndarray requires a copy" + ) + # obj is a scalar, create 0d array + return _asarray_from_numpy_ndarray( + np.asarray(obj), + dtype=dtype, + usm_type=usm_type, + sycl_queue=sycl_queue, + order="C", + ) + + +def empty( + sh, dtype="f8", order="C", device=None, usm_type="device", sycl_queue=None +): + """ + Creates `usm_ndarray` from uninitialized USM allocation. + + Args: + shape (tuple): Dimensions of the array to be created. + dtype (optional): data type of the array. Can be typestring, + a `numpy.dtype` object, `numpy` char string, or a numpy + scalar type. Default: "f8" + order ("C", or F"): memory layout for the array. Default: "C" + device (optional): array API concept of device where the output array + is created. `device` can be `None`, a oneAPI filter selector string, + an instance of :class:`dpctl.SyclDevice` corresponding to a + non-partitioned SYCL device, an instance of + :class:`dpctl.SyclQueue`, or a `Device` object returnedby + `dpctl.tensor.usm_array.device`. Default: `None`. + usm_type ("device"|"shared"|"host", optional): The type of SYCL USM + allocation for the output array. Default: `"device"`. + sycl_queue: (:class:`dpctl.SyclQueue`, optional): The SYCL queue to use + for output array allocation and copying. `sycl_queue` and `device` + are exclusive keywords, i.e. use one or another. If both are + specified, a `TypeError` is raised unless both imply the same + underlying SYCL queue to be used. If both a `None`, the + `dpctl.SyclQueue()` is used for allocation and copying. + Default: `None`. + """ + dtype = np.dtype(dtype) + if not isinstance(order, str) or len(order) == 0 or order[0] not in "CcFf": + raise ValueError( + "Unrecognized order keyword value, expecting 'F' or 'C'." + ) + else: + order = order[0].upper() + if isinstance(usm_type, str): + if usm_type not in ["device", "shared", "host"]: + raise ValueError( + f"Unrecognized value of usm_type={usm_type}, " + "expected 'device', 'shared', or 'host'." + ) + else: + raise TypeError( + f"Expected usm_type to be of type str, got {type(usm_type)}" + ) + sycl_queue = normalize_queue_device(sycl_queue=sycl_queue, device=device) + res = dpt.usm_ndarray( + sh, + dtype=dtype, + buffer=usm_type, + order=order, + buffer_ctor_kwargs={"queue": sycl_queue}, + ) + return res diff --git a/dpctl/tensor/_device.py b/dpctl/tensor/_device.py index 7959735b4e..c8f8bdef1b 100644 --- a/dpctl/tensor/_device.py +++ b/dpctl/tensor/_device.py @@ -35,8 +35,6 @@ def __new__(cls, *args, **kwargs): @classmethod def create_device(cls, dev): """ - Device.create_device(device) - Creates instance of Device from argument. Args: @@ -105,3 +103,54 @@ def __repr__(self): except TypeError: # This is a sub-device return repr(self.sycl_queue) + + +def normalize_queue_device(sycl_queue=None, device=None): + """ + normalize_queue_device(sycl_queue=None, device=None) + + Utility to process exclusive keyword arguments 'device' + and 'sycl_queue' in functions of `dpctl.tensor`. + + Args: + sycl_queue(:class:`dpctl.SyclQueue`, optional): + explicitly indicates where USM allocation is done + and the population code (if any) is executed. + Value `None` is interpreted as get the SYCL queue + from `device` keyword, or use default queue. + Default: None + device (string, :class:`dpctl.SyclDevice`, :class:`dpctl.SyclQueue, + :class:`dpctl.tensor.Device`, optional): + array-API keyword indicating non-partitioned SYCL device + where array is allocated. + Returns + :class:`dpctl.SyclQueue` object implied by either of provided + keywords. If both are None, `dpctl.SyclQueue()` is returned. + If both are specified and imply the same queue, `sycl_queue` + is returned. + Raises: + TypeError: if argument is not of the expected type, or keywords + imply incompatible queues. + """ + q = sycl_queue + d = device + if q is None: + d = Device.create_device(d) + return d.sycl_queue + else: + if not isinstance(q, dpctl.SyclQueue): + raise TypeError(f"Expected dpctl.SyclQueue, got {type(q)}") + if d is None: + return q + d = Device.create_device(d) + qq = dpctl.utils.get_execution_queue( + ( + q, + d.sycl_queue, + ) + ) + if qq is None: + raise TypeError( + "sycl_queue and device keywords can not be both specified" + ) + return qq diff --git a/dpctl/tensor/_dlpack.pxd b/dpctl/tensor/_dlpack.pxd new file mode 100644 index 0000000000..58176ceb84 --- /dev/null +++ b/dpctl/tensor/_dlpack.pxd @@ -0,0 +1,41 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +from ._usmarray cimport usm_ndarray + + +cdef extern from 'dlpack/dlpack.h' nogil: + int device_CPU 'kDLCPU' + int device_oneAPI 'kDLOneAPI' + int device_OpenCL 'kDLOpenCL' + + +cpdef object to_dlpack_capsule(usm_ndarray array) except + +cpdef usm_ndarray from_dlpack_capsule(object dltensor) except + + +cpdef from_dlpack(array) + +cdef class DLPackCreationError(Exception): + """ + A DLPackCreateError exception is raised when constructing + DLPack capsule from `usm_ndarray` based on a USM allocation + on a partitioned SYCL device. + """ + pass diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx new file mode 100644 index 0000000000..fe762a016a --- /dev/null +++ b/dpctl/tensor/_dlpack.pyx @@ -0,0 +1,437 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# distutils: language = c++ +# cython: language_level=3 +# cython: linetrace=True + +cimport cpython +from libc cimport stdlib +from libc.stdint cimport int32_t, int64_t, uint8_t, uint16_t, uint64_t + +cimport dpctl as c_dpctl +cimport dpctl.memory as c_dpmem + +from .._backend cimport ( + DPCTLDevice_Delete, + DPCTLDevice_GetParentDevice, + DPCTLSyclDeviceRef, + DPCTLSyclUSMRef, +) +from ._usmarray cimport usm_ndarray + +import numpy as np + +import dpctl +import dpctl.memory as dpmem + + +cdef extern from 'dlpack/dlpack.h' nogil: + cdef int DLPACK_VERSION + + cdef enum DLDeviceType: + kDLCPU + kDLCUDA + kDLCUDAHost + kDLCUDAManaged + kDLROCM + kDLROCMHost + kDLOpenCL + kDLVulkan + kDLMetal + kDLVPI + kDLOneAPI + + ctypedef struct DLDevice: + DLDeviceType device_type + int device_id + + cdef enum DLDataTypeCode: + kDLInt + kDLUInt + kDLFloat + kDLBfloat + kDLComplex + + ctypedef struct DLDataType: + uint8_t code + uint8_t bits + uint16_t lanes + + ctypedef struct DLTensor: + void *data + DLDevice device + int ndim + DLDataType dtype + int64_t *shape + int64_t *strides + uint64_t byte_offset + + ctypedef struct DLManagedTensor: + DLTensor dl_tensor + void *manager_ctx + void (*deleter)(DLManagedTensor *) # noqa: E211 + + +def get_build_dlpack_version(): + """ + Returns the string value of DLPACK_VERSION from dlpack.h + `dpctl.tensor` was built with. + + Returns: + A string value of the version of DLPack used to build + `dpctl`. + """ + return str(DLPACK_VERSION) + + +cdef void _pycapsule_deleter(object dlt_capsule): + cdef DLManagedTensor *dlm_tensor = NULL + if cpython.PyCapsule_IsValid(dlt_capsule, 'dltensor'): + dlm_tensor = cpython.PyCapsule_GetPointer( + dlt_capsule, 'dltensor') + dlm_tensor.deleter(dlm_tensor) + + +cdef void _managed_tensor_deleter(DLManagedTensor *dlm_tensor) with gil: + if dlm_tensor is not NULL: + stdlib.free(dlm_tensor.dl_tensor.shape) + cpython.Py_DECREF(dlm_tensor.manager_ctx) + dlm_tensor.manager_ctx = NULL + stdlib.free(dlm_tensor) + + +cpdef to_dlpack_capsule(usm_ndarray usm_ary) except+: + """ + to_dlpack_capsule(usm_ary) + + Constructs named Python capsule object referencing + instance of `DLManagerTensor` from + :class:`dpctl.tensor.usm_ndarray` instance. + + Args: + usm_ary: An instance of :class:`dpctl.tensor.usm_ndarray` + Returns: + Python a new capsule with name "dltensor" that contains + a pointer to `DLManagedTensor` struct. + Raises: + DLPackCreationError: when array can be represented as + DLPack tensor. This may happen when array was allocated + on a partitioned sycl device, or its USM allocation is + not bound to the platform default SYCL context. + MemoryError: when host allocation to needed for `DLManagedTensor` + did not succeed. + ValueError: when array elements data type could not be represented + in `DLManagedTensor`. + """ + cdef c_dpctl.SyclQueue ary_sycl_queue + cdef c_dpctl.SyclDevice ary_sycl_device + cdef DPCTLSyclDeviceRef pDRef = NULL + cdef DLManagedTensor *dlm_tensor = NULL + cdef DLTensor *dl_tensor = NULL + cdef int nd = usm_ary.get_ndim() + cdef char *data_ptr = usm_ary.get_data() + cdef Py_ssize_t *shape_ptr = NULL + cdef Py_ssize_t *strides_ptr = NULL + cdef int64_t *shape_strides_ptr = NULL + cdef int i = 0 + cdef int device_id = -1 + cdef char *base_ptr = NULL + cdef Py_ssize_t element_offset = 0 + cdef Py_ssize_t byte_offset = 0 + + ary_base = usm_ary.get_base() + ary_sycl_queue = usm_ary.get_sycl_queue() + ary_sycl_device = ary_sycl_queue.get_sycl_device() + + # check that ary_sycl_device is a non-partitioned device + pDRef = DPCTLDevice_GetParentDevice(ary_sycl_device.get_device_ref()) + if pDRef is not NULL: + DPCTLDevice_Delete(pDRef) + raise DLPackCreationError( + "to_dlpack_capsule: DLPack can only export arrays allocated on " + "non-partitioned SYCL devices." + ) + default_context = dpctl.SyclQueue(ary_sycl_device).sycl_context + if not usm_ary.sycl_context == default_context: + raise DLPackCreationError( + "to_dlpack_capsule: DLPack can only export arrays based on USM " + "allocations bound to a default platform SYCL context" + ) + + dlm_tensor = stdlib.malloc( + sizeof(DLManagedTensor)) + if dlm_tensor is NULL: + raise MemoryError( + "to_dlpack_capsule: Could not allocate memory for DLManagedTensor" + ) + shape_strides_ptr = stdlib.malloc((sizeof(int64_t) * 2) * nd) + if shape_strides_ptr is NULL: + stdlib.free(dlm_tensor) + raise MemoryError( + "to_dlpack_capsule: Could not allocate memory for shape/strides" + ) + shape_ptr = usm_ary.get_shape() + for i in range(nd): + shape_strides_ptr[i] = shape_ptr[i] + strides_ptr = usm_ary.get_strides() + if strides_ptr: + for i in range(nd): + shape_strides_ptr[nd + i] = strides_ptr[i] + + device_id = ary_sycl_device.get_overall_ordinal() + if device_id < 0: + stdlib.free(shape_strides_ptr) + stdlib.free(dlm_tensor) + raise DLPackCreationError( + "to_dlpack_capsule: failed to determine device_id" + ) + + ary_dt = usm_ary.dtype + ary_dtk = ary_dt.kind + element_offset = usm_ary.get_offset() + byte_offset = element_offset * (ary_dt.itemsize) + + dl_tensor = &dlm_tensor.dl_tensor + dl_tensor.data = (data_ptr - byte_offset) + dl_tensor.ndim = nd + dl_tensor.byte_offset = byte_offset + dl_tensor.shape = &shape_strides_ptr[0] + if strides_ptr is NULL: + dl_tensor.strides = NULL + else: + dl_tensor.strides = &shape_strides_ptr[nd] + dl_tensor.device.device_type = kDLOneAPI + dl_tensor.device.device_id = device_id + dl_tensor.dtype.lanes = 1 + dl_tensor.dtype.bits = (ary_dt.itemsize * 8) + if (ary_dtk == "b"): + dl_tensor.dtype.code = kDLUInt + elif (ary_dtk == "u"): + dl_tensor.dtype.code = kDLUInt + elif (ary_dtk == "i"): + dl_tensor.dtype.code = kDLInt + elif (ary_dtk == "f"): + dl_tensor.dtype.code = kDLFloat + elif (ary_dtk == "c"): + dl_tensor.dtype.code = kDLComplex + else: + stdlib.free(shape_strides_ptr) + stdlib.free(dlm_tensor) + raise ValueError("Unrecognized array data type") + + dlm_tensor.manager_ctx = usm_ary + cpython.Py_INCREF(usm_ary) + dlm_tensor.deleter = _managed_tensor_deleter + + return cpython.PyCapsule_New(dlm_tensor, 'dltensor', _pycapsule_deleter) + + +cdef class _DLManagedTensorOwner: + """ + Helper class managing the lifetime of the DLManagedTensor struct + transferred from a 'dlpack' capsule. + """ + cdef DLManagedTensor *dlm_tensor + + def __cinit__(self): + self.dlm_tensor = NULL + + def __dealloc__(self): + if self.dlm_tensor: + self.dlm_tensor.deleter(self.dlm_tensor) + + @staticmethod + cdef _DLManagedTensorOwner _create(DLManagedTensor *dlm_tensor_src): + cdef _DLManagedTensorOwner res = _DLManagedTensorOwner.__new__(_DLManagedTensorOwner) + res.dlm_tensor = dlm_tensor_src + return res + + +cpdef usm_ndarray from_dlpack_capsule(object py_caps) except +: + """ + from_dlpack_capsule(caps) + + Reconstructs instance of :class:`dpctl.tensor.usm_ndarray` from + named Python capsule object referencing instance of `DLManagedTensor` + without copy. The instance forms a view in the memory of the tensor. + + Args: + caps: Python capsule with name "dltensor" expected to reference + an instance of `DLManagedTensor` struct. + Returns: + Instance of :class:`dpctl.tensor.usm_ndarray` with a view into + memory of the tensor. Capsule is renamed to "used_dltensor" upon + success. + Raises: + TypeError: if argument is not a "dltensor" capsule. + ValueError: if argument is "used_dltensor" capsule, + if the USM pointer is not bound to the reconstructed + sycl context, or the DLPack's device_type is not supported + by dpctl. + """ + cdef DLManagedTensor *dlm_tensor = NULL + cdef bytes usm_type + cdef size_t sz = 1 + cdef int i + cdef int element_bytesize = 0 + cdef Py_ssize_t offset_min = 0 + cdef Py_ssize_t offset_max = 0 + cdef int64_t stride_i + cdef char *mem_ptr = NULL + cdef Py_ssize_t element_offset = 0 + + if not cpython.PyCapsule_IsValid(py_caps, 'dltensor'): + if cpython.PyCapsule_IsValid(py_caps, 'used_dltensor'): + raise ValueError( + "A DLPack tensor object can not be consumed multiple times" + ) + else: + raise TypeError( + f"A Python 'dltensor' capsule was expected, " + "got {type(dlm_tensor)}" + ) + dlm_tensor = cpython.PyCapsule_GetPointer( + py_caps, "dltensor") + # Verify that we can work with this device + if dlm_tensor.dl_tensor.device.device_type == kDLOneAPI: + q = dpctl.SyclQueue(str(dlm_tensor.dl_tensor.device.device_id)) + if dlm_tensor.dl_tensor.data is NULL: + usm_type = b"device" + else: + usm_type = c_dpmem._Memory.get_pointer_type( + dlm_tensor.dl_tensor.data, + q.sycl_context) + if usm_type == b"unknown": + raise ValueError( + f"Data pointer in DLPack is not bound to default sycl " + "context of device '{device_id}', translated to " + "{q.sycl_device.filter_string}" + ) + if dlm_tensor.dl_tensor.dtype.bits % 8: + raise ValueError( + "Can not import DLPack tensor whose element's " + "bitsize is not a multiple of 8" + ) + if dlm_tensor.dl_tensor.dtype.lanes != 1: + raise ValueError( + "Can not import DLPack tensor with lanes != 1" + ) + if dlm_tensor.dl_tensor.strides is NULL: + for i in range(dlm_tensor.dl_tensor.ndim): + sz = sz * dlm_tensor.dl_tensor.shape[i] + else: + offset_min = 0 + offset_max = 0 + for i in range(dlm_tensor.dl_tensor.ndim): + stride_i = dlm_tensor.dl_tensor.strides[i] + if stride_i > 0: + offset_max = offset_max + stride_i * ( + dlm_tensor.dl_tensor.shape[i] - 1 + ) + else: + offset_min = offset_min + stride_i * ( + dlm_tensor.dl_tensor.shape[i] - 1 + ) + sz = offset_max - offset_min + 1 + if sz == 0: + sz = 1 + + element_bytesize = (dlm_tensor.dl_tensor.dtype.bits // 8) + sz = sz * element_bytesize + element_offset = dlm_tensor.dl_tensor.byte_offset // element_bytesize + + # transfer dlm_tensor ownership + dlm_holder = _DLManagedTensorOwner._create(dlm_tensor) + cpython.PyCapsule_SetName(py_caps, 'used_dltensor') + + if dlm_tensor.dl_tensor.data is NULL: + usm_mem = dpmem.MemoryUSMDevice(sz, q) + else: + mem_ptr = dlm_tensor.dl_tensor.data + dlm_tensor.dl_tensor.byte_offset + mem_ptr = mem_ptr - (element_offset * element_bytesize) + usm_mem = c_dpmem._Memory.create_from_usm_pointer_size_qref( + mem_ptr, + sz, + (q).get_queue_ref(), + memory_owner=dlm_holder + ) + py_shape = list() + for i in range(dlm_tensor.dl_tensor.ndim): + py_shape.append(dlm_tensor.dl_tensor.shape[i]) + if (dlm_tensor.dl_tensor.strides is NULL): + py_strides = None + else: + py_strides = list() + for i in range(dlm_tensor.dl_tensor.ndim): + py_strides.append(dlm_tensor.dl_tensor.strides[i]) + if (dlm_tensor.dl_tensor.dtype.code == kDLUInt): + ary_dt = np.dtype("u" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLInt): + ary_dt = np.dtype("i" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLFloat): + ary_dt = np.dtype("f" + str(element_bytesize)) + elif (dlm_tensor.dl_tensor.dtype.code == kDLComplex): + ary_dt = np.dtype("c" + str(element_bytesize)) + else: + raise ValueError( + "Can not import DLPack tensor with type code {}.".format( + dlm_tensor.dl_tensor.dtype.code + ) + ) + res_ary = usm_ndarray( + py_shape, + dtype=ary_dt, + buffer=usm_mem, + strides=py_strides, + offset=element_offset + ) + return res_ary + else: + raise ValueError( + "The DLPack tensor resides on unsupported device." + ) + + +cpdef from_dlpack(array): + """ + dpctl.tensor.from_dlpack(obj) + + Constructs :class:`dpctl.tensor.usm_ndarray` instance from a Python + object `obj` that implements `__dlpack__` protocol. The output + array is always a zero-copy view of the input. + + Args: + A Python object representing an array that supports `__dlpack__` + protocol. + Raises: + TypeError: if `obj` does not implement `__dlpack__` method. + ValueError: if zero copy view can not be constructed because + the input array resides on an unsupported device. + """ + if not hasattr(array, "__dlpack__"): + raise TypeError( + "The argument of type {type(array)} does not implement " + "`__dlpack__` method." + ) + dlpack_attr = getattr(array, "__dlpack__") + if not callable(dlpack_attr): + raise TypeError( + "The argument of type {type(array)} does not implement " + "`__dlpack__` method." + ) + dlpack_capsule = dlpack_attr() + return from_dlpack_capsule(dlpack_capsule) diff --git a/dpctl/tensor/_reshape.py b/dpctl/tensor/_reshape.py index 8e04f1b20c..ffa60c3652 100644 --- a/dpctl/tensor/_reshape.py +++ b/dpctl/tensor/_reshape.py @@ -104,7 +104,10 @@ def reshape(X, newshape, order="C"): newshape = [v if d == -1 else d for d in newshape] if X.size != np.prod(newshape): raise ValueError("Can not reshape into {}".format(newshape)) - newsts = reshaped_strides(X.shape, X.strides, newshape, order=order) + if X.size: + newsts = reshaped_strides(X.shape, X.strides, newshape, order=order) + else: + newsts = (1,) * len(newshape) if newsts is None: # must perform a copy flat_res = dpt.usm_ndarray( diff --git a/dpctl/tensor/_stride_utils.pxi b/dpctl/tensor/_stride_utils.pxi index 37d5a366b7..c019cd8ddd 100644 --- a/dpctl/tensor/_stride_utils.pxi +++ b/dpctl/tensor/_stride_utils.pxi @@ -61,6 +61,7 @@ cdef int _from_input_shape_strides( Otherwise they are set to NULL """ cdef int i + cdef int j cdef int all_incr = 1 cdef int all_decr = 1 cdef Py_ssize_t elem_count = 1 @@ -115,6 +116,15 @@ cdef int _from_input_shape_strides( contig[0] = USM_ARRAY_C_CONTIGUOUS else: contig[0] = USM_ARRAY_F_CONTIGUOUS + if nd == 1: + contig[0] = USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS + else: + j = 0 + for i in range(nd): + if shape_arr[i] > 1: + j = j + 1 + if j < 2: + contig[0] = USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS min_disp[0] = 0 max_disp[0] = (elem_count - 1) strides_ptr[0] = (0) @@ -137,26 +147,42 @@ cdef int _from_input_shape_strides( min_disp[0] = min_shift max_disp[0] = max_shift if max_shift == min_shift + (elem_count - 1): + if elem_count == 1: + contig[0] = (USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS) + return 0 if nd == 1: if strides_arr[0] == 1: contig[0] = USM_ARRAY_C_CONTIGUOUS else: contig[0] = 0 return 0 - for i in range(0, nd - 1): - if all_incr: - all_incr = ( - (strides_arr[i] > 0) and - (strides_arr[i+1] > 0) and - (strides_arr[i] <= strides_arr[i + 1]) - ) - if all_decr: - all_decr = ( - (strides_arr[i] > 0) and - (strides_arr[i+1] > 0) and - (strides_arr[i] >= strides_arr[i + 1]) - ) - if all_incr: + i = 0 + while i < nd: + if shape_arr[i] == 1: + i = i + 1 + continue + j = i + 1 + while (j < nd and shape_arr[j] == 1): + j = j + 1 + if j < nd: + if all_incr: + all_incr = ( + (strides_arr[i] > 0) and + (strides_arr[j] > 0) and + (strides_arr[i] <= strides_arr[j]) + ) + if all_decr: + all_decr = ( + (strides_arr[i] > 0) and + (strides_arr[j] > 0) and + (strides_arr[i] >= strides_arr[j]) + ) + i = j + else: + break + if all_incr and all_decr: + contig[0] = (USM_ARRAY_C_CONTIGUOUS | USM_ARRAY_F_CONTIGUOUS) + elif all_incr: contig[0] = USM_ARRAY_F_CONTIGUOUS elif all_decr: contig[0] = USM_ARRAY_C_CONTIGUOUS diff --git a/dpctl/tensor/_usmarray.pxd b/dpctl/tensor/_usmarray.pxd index e0d53b05af..4a7997fa94 100644 --- a/dpctl/tensor/_usmarray.pxd +++ b/dpctl/tensor/_usmarray.pxd @@ -4,27 +4,27 @@ cimport dpctl -cdef public int USM_ARRAY_C_CONTIGUOUS -cdef public int USM_ARRAY_F_CONTIGUOUS -cdef public int USM_ARRAY_WRITEABLE - -cdef public int UAR_BOOL -cdef public int UAR_BYTE -cdef public int UAR_UBYTE -cdef public int UAR_SHORT -cdef public int UAR_USHORT -cdef public int UAR_INT -cdef public int UAR_UINT -cdef public int UAR_LONG -cdef public int UAR_ULONG -cdef public int UAR_LONGLONG -cdef public int UAR_ULONGLONG -cdef public int UAR_FLOAT -cdef public int UAR_DOUBLE -cdef public int UAR_CFLOAT -cdef public int UAR_CDOUBLE -cdef public int UAR_TYPE_SENTINEL -cdef public int UAR_HALF +cdef public api int USM_ARRAY_C_CONTIGUOUS +cdef public api int USM_ARRAY_F_CONTIGUOUS +cdef public api int USM_ARRAY_WRITEABLE + +cdef public api int UAR_BOOL +cdef public api int UAR_BYTE +cdef public api int UAR_UBYTE +cdef public api int UAR_SHORT +cdef public api int UAR_USHORT +cdef public api int UAR_INT +cdef public api int UAR_UINT +cdef public api int UAR_LONG +cdef public api int UAR_ULONG +cdef public api int UAR_LONGLONG +cdef public api int UAR_ULONGLONG +cdef public api int UAR_FLOAT +cdef public api int UAR_DOUBLE +cdef public api int UAR_CFLOAT +cdef public api int UAR_CDOUBLE +cdef public api int UAR_TYPE_SENTINEL +cdef public api int UAR_HALF cdef api class usm_ndarray [object PyUSMArrayObject, type PyUSMArrayType]: diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 57c5225a3e..b1ce20a1b2 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -32,6 +32,7 @@ from cpython.tuple cimport PyTuple_New, PyTuple_SetItem cimport dpctl as c_dpctl cimport dpctl.memory as c_dpmem +cimport dpctl.tensor._dlpack as c_dlpack include "_stride_utils.pxi" include "_types.pxi" @@ -99,14 +100,7 @@ cdef class InternalUSMArrayError(Exception): cdef class usm_ndarray: - """ - usm_ndarray( - shape, dtype="|f8", strides=None, buffer='device', - offset=0, order='C', - buffer_ctor_kwargs=dict(), - array_namespace=None - ) - + """ usm_ndarray(shape, dtype="|f8", strides=None, buffer="device", offset=0, order="C", buffer_ctor_kwargs=dict(), array_namespace=None) See :class:`dpctl.memory.MemoryUSMShared` for allowed keyword arguments. @@ -434,6 +428,7 @@ cdef class usm_ndarray: cdef int contig_flag = 0 cdef Py_ssize_t *shape_ptr = NULL cdef Py_ssize_t *strides_ptr = NULL + cdef Py_ssize_t size = -1 import operator from ._reshape import reshaped_strides @@ -445,15 +440,19 @@ cdef class usm_ndarray: raise TypeError( "Target shape must be a finite iterable of integers" ) - if not np.prod(new_shape) == shape_to_elem_count(self.nd_, self.shape_): + size = shape_to_elem_count(self.nd_, self.shape_) + if not np.prod(new_shape) == size: raise TypeError( f"Can not reshape array of size {self.size} into {new_shape}" ) - new_strides = reshaped_strides( - self.shape, - self.strides, - new_shape - ) + if size > 0: + new_strides = reshaped_strides( + self.shape, + self.strides, + new_shape + ) + else: + new_strides = (1,) * len(new_shape) if new_strides is None: raise AttributeError( "Incompatible shape for in-place modification. " @@ -738,10 +737,46 @@ cdef class usm_ndarray: return NotImplemented def __dlpack__(self, stream=None): - return NotImplemented + """ + Produces DLPack capsule. + + Raises: + MemoryError: when host memory can not be allocated. + DLPackCreationError: when array is allocated on a partitioned + SYCL device, or with a non-default context. + NotImplementedError: when non-default value of `stream` keyword + is used. + """ + if stream is None: + return c_dlpack.to_dlpack_capsule(self) + else: + raise NotImplementedError( + "Only stream=None is supported. " + "Use `dpctl.SyclQueue.submit_barrier` to synchronize queues." + ) def __dlpack_device__(self): - return NotImplemented + """ + Gives a tuple (`device_type`, `device_id`) corresponding to `DLDevice` + entry in `DLTensor` in DLPack protocol. + + The tuple describes the non-partitioned device where the array + has been allocated. + + Raises: + DLPackCreationError: when array is allocation on a partitioned + SYCL device + """ + cdef int dev_id = (self.sycl_device).get_overall_ordinal() + if dev_id < 0: + raise c_dlpack.DLPackCreationError( + "DLPack protocol is only supported for non-partitioned devices" + ) + else: + return ( + c_dlpack.device_oneAPI, + dev_id, + ) def __eq__(self, other): return _dispatch_binary_elementwise(self, "equal", other) @@ -822,7 +857,7 @@ cdef class usm_ndarray: return NotImplemented def __pos__(self): - return _dispatch_unary_elementwise(self, "positive") + return self # _dispatch_unary_elementwise(self, "positive") def __pow__(first, other, mod): "See comment in __add__" @@ -847,13 +882,13 @@ cdef class usm_ndarray: except (ValueError, IndexError) as e: raise e from ._copy_utils import ( - copy_from_numpy_into, - copy_from_usm_ndarray_to_usm_ndarray, + _copy_from_numpy_into, + _copy_from_usm_ndarray_to_usm_ndarray, ) if isinstance(val, usm_ndarray): - copy_from_usm_ndarray_to_usm_ndarray(Xv, val) + _copy_from_usm_ndarray_to_usm_ndarray(Xv, val) else: - copy_from_numpy_into(Xv, np.asarray(val)) + _copy_from_numpy_into(Xv, np.asarray(val)) def __sub__(first, other): "See comment in __add__" @@ -1106,37 +1141,42 @@ cdef usm_ndarray _zero_like(usm_ndarray ary): return r -cdef api char* usm_ndarray_get_data(usm_ndarray arr): - """ - """ +cdef api char* UsmNDArray_GetData(usm_ndarray arr): + """Get allocation pointer of zero index element of array """ return arr.get_data() -cdef api int usm_ndarray_get_ndim(usm_ndarray arr): - """""" +cdef api int UsmNDArray_GetNDim(usm_ndarray arr): + """Get array rank: length of its shape""" return arr.get_ndim() -cdef api Py_ssize_t* usm_ndarray_get_shape(usm_ndarray arr): - """ """ +cdef api Py_ssize_t* UsmNDArray_GetShape(usm_ndarray arr): + """Get host pointer to shape vector""" return arr.get_shape() -cdef api Py_ssize_t* usm_ndarray_get_strides(usm_ndarray arr): - """ """ +cdef api Py_ssize_t* UsmNDArray_GetStrides(usm_ndarray arr): + """Get host pointer to strides vector""" return arr.get_strides() -cdef api int usm_ndarray_get_typenum(usm_ndarray arr): - """ """ +cdef api int UsmNDArray_GetTypenum(usm_ndarray arr): + """Get type number for data type of array elements""" return arr.get_typenum() -cdef api int usm_ndarray_get_flags(usm_ndarray arr): - """ """ +cdef api int UsmNDArray_GetFlags(usm_ndarray arr): + """Get flags of array""" return arr.get_flags() -cdef api c_dpctl.DPCTLSyclQueueRef usm_ndarray_get_queue_ref(usm_ndarray arr): - """ """ +cdef api c_dpctl.DPCTLSyclQueueRef UsmNDArray_GetQueueRef(usm_ndarray arr): + """Get DPCTLSyclQueueRef for queue associated with the array""" return arr.get_queue_ref() + + +cdef api Py_ssize_t UsmNDArray_GetOffset(usm_ndarray arr): + """Get offset of zero-index array element from the beginning of the USM + allocation.""" + return arr.get_offset() diff --git a/dpctl/tensor/include/dlpack/.clang-format b/dpctl/tensor/include/dlpack/.clang-format new file mode 100644 index 0000000000..9d159247d5 --- /dev/null +++ b/dpctl/tensor/include/dlpack/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: false diff --git a/dpctl/tensor/include/dlpack/LICENSE.third-party b/dpctl/tensor/include/dlpack/LICENSE.third-party new file mode 100644 index 0000000000..20a9c8a7b4 --- /dev/null +++ b/dpctl/tensor/include/dlpack/LICENSE.third-party @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dpctl/tensor/include/dlpack/README.md b/dpctl/tensor/include/dlpack/README.md new file mode 100644 index 0000000000..5d9bf51177 --- /dev/null +++ b/dpctl/tensor/include/dlpack/README.md @@ -0,0 +1,7 @@ +# DLPack header + +The header `dlpack.h` downloaded from `https://github.com/dmlc/dlpack.git` remote at commit [`98861a50e5`](https://github.com/dmlc/dlpack/commit/98861a50e5ade5a6b2df388b12d67b418e3baebe). + +The file can also be viewed using github web interface at https://github.com/dmlc/dlpack/blob/98861a50e5ade5a6b2df388b12d67b418e3baebe/include/dlpack/dlpack.h + +License file was retrived from https://github.com/dmlc/dlpack/blob/main/LICENSE diff --git a/dpctl/tensor/include/dlpack/dlpack.h b/dpctl/tensor/include/dlpack/dlpack.h new file mode 100644 index 0000000000..afbac0573a --- /dev/null +++ b/dpctl/tensor/include/dlpack/dlpack.h @@ -0,0 +1,213 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file dlpack.h + * \brief The common header of DLPack. + */ +#ifndef DLPACK_DLPACK_H_ +#define DLPACK_DLPACK_H_ + +#ifdef __cplusplus +#define DLPACK_EXTERN_C extern "C" +#else +#define DLPACK_EXTERN_C +#endif + +/*! \brief The current version of dlpack */ +#define DLPACK_VERSION 60 + +/*! \brief DLPACK_DLL prefix for windows */ +#ifdef _WIN32 +#ifdef DLPACK_EXPORTS +#define DLPACK_DLL __declspec(dllexport) +#else +#define DLPACK_DLL __declspec(dllimport) +#endif +#else +#define DLPACK_DLL +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +/*! + * \brief The device type in DLDevice. + */ +typedef enum { + /*! \brief CPU device */ + kDLCPU = 1, + /*! \brief CUDA GPU device */ + kDLCUDA = 2, + /*! + * \brief Pinned CUDA CPU memory by cudaMallocHost + */ + kDLCUDAHost = 3, + /*! \brief OpenCL devices. */ + kDLOpenCL = 4, + /*! \brief Vulkan buffer for next generation graphics. */ + kDLVulkan = 7, + /*! \brief Metal for Apple GPU. */ + kDLMetal = 8, + /*! \brief Verilog simulator buffer */ + kDLVPI = 9, + /*! \brief ROCm GPUs for AMD GPUs */ + kDLROCM = 10, + /*! + * \brief Pinned ROCm CPU memory allocated by hipMallocHost + */ + kDLROCMHost = 11, + /*! + * \brief Reserved extension device type, + * used for quickly test extension device + * The semantics can differ depending on the implementation. + */ + kDLExtDev = 12, + /*! + * \brief CUDA managed/unified memory allocated by cudaMallocManaged + */ + kDLCUDAManaged = 13, + /*! + * \brief Unified shared memory allocated on a oneAPI non-partititioned + * device. Call to oneAPI runtime is required to determine the device + * type, the USM allocation type and the sycl context it is bound to. + * + */ + kDLOneAPI = 14, +} DLDeviceType; + +/*! + * \brief A Device for Tensor and operator. + */ +typedef struct { + /*! \brief The device type used in the device. */ + DLDeviceType device_type; + /*! + * \brief The device index. + * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0. + */ + int device_id; +} DLDevice; + +/*! + * \brief The type code options DLDataType. + */ +typedef enum { + /*! \brief signed integer */ + kDLInt = 0U, + /*! \brief unsigned integer */ + kDLUInt = 1U, + /*! \brief IEEE floating point */ + kDLFloat = 2U, + /*! + * \brief Opaque handle type, reserved for testing purposes. + * Frameworks need to agree on the handle data type for the exchange to be well-defined. + */ + kDLOpaqueHandle = 3U, + /*! \brief bfloat16 */ + kDLBfloat = 4U, + /*! + * \brief complex number + * (C/C++/Python layout: compact struct per complex number) + */ + kDLComplex = 5U, +} DLDataTypeCode; + +/*! + * \brief The data type the tensor can hold. + * + * Examples + * - float: type_code = 2, bits = 32, lanes=1 + * - float4(vectorized 4 float): type_code = 2, bits = 32, lanes=4 + * - int8: type_code = 0, bits = 8, lanes=1 + * - std::complex: type_code = 5, bits = 64, lanes = 1 + */ +typedef struct { + /*! + * \brief Type code of base types. + * We keep it uint8_t instead of DLDataTypeCode for minimal memory + * footprint, but the value should be one of DLDataTypeCode enum values. + * */ + uint8_t code; + /*! + * \brief Number of bits, common choices are 8, 16, 32. + */ + uint8_t bits; + /*! \brief Number of lanes in the type, used for vector types. */ + uint16_t lanes; +} DLDataType; + +/*! + * \brief Plain C Tensor object, does not manage memory. + */ +typedef struct { + /*! + * \brief The data pointer points to the allocated data. This will be CUDA + * device pointer or cl_mem handle in OpenCL. It may be opaque on some device + * types. This pointer is always aligned to 256 bytes as in CUDA. The + * `byte_offset` field should be used to point to the beginning of the data. + * + * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow, + * TVM, perhaps others) do not adhere to this 256 byte aligment requirement + * on CPU/CUDA/ROCm, and always use `byte_offset=0`. This must be fixed + * (after which this note will be updated); at the moment it is recommended + * to not rely on the data pointer being correctly aligned. + * + * For given DLTensor, the size of memory required to store the contents of + * data is calculated as follows: + * + * \code{.c} + * static inline size_t GetDataSize(const DLTensor* t) { + * size_t size = 1; + * for (tvm_index_t i = 0; i < t->ndim; ++i) { + * size *= t->shape[i]; + * } + * size *= (t->dtype.bits * t->dtype.lanes + 7) / 8; + * return size; + * } + * \endcode + */ + void* data; + /*! \brief The device of the tensor */ + DLDevice device; + /*! \brief Number of dimensions */ + int ndim; + /*! \brief The data type of the pointer*/ + DLDataType dtype; + /*! \brief The shape of the tensor */ + int64_t* shape; + /*! + * \brief strides of the tensor (in number of elements, not bytes) + * can be NULL, indicating tensor is compact and row-majored. + */ + int64_t* strides; + /*! \brief The offset in bytes to the beginning pointer to data */ + uint64_t byte_offset; +} DLTensor; + +/*! + * \brief C Tensor object, manage memory of DLTensor. This data structure is + * intended to facilitate the borrowing of DLTensor by another framework. It is + * not meant to transfer the tensor. When the borrowing framework doesn't need + * the tensor, it should call the deleter to notify the host that the resource + * is no longer needed. + */ +typedef struct DLManagedTensor { + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; + /*! \brief the context of the original host framework of DLManagedTensor in + * which DLManagedTensor is used in the framework. It can also be NULL. + */ + void * manager_ctx; + /*! \brief Destructor signature void (*)(void*) - this should be called + * to destruct manager_ctx which holds the DLManagedTensor. It can be NULL + * if there is no way for the caller to provide a reasonable destructor. + * The destructors deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensor * self); +} DLManagedTensor; +#ifdef __cplusplus +} // DLPACK_EXTERN_C +#endif +#endif // DLPACK_DLPACK_H_ diff --git a/dpctl/tests/__init__.py b/dpctl/tests/conftest.py similarity index 82% rename from dpctl/tests/__init__.py rename to dpctl/tests/conftest.py index 2fc36c98cd..6b85eea77c 100644 --- a/dpctl/tests/__init__.py +++ b/dpctl/tests/conftest.py @@ -14,5 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Top-level module of all dpctl Python unit test cases. +""" Configures pytest to discover helper/ module """ + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "helper")) diff --git a/dpctl/tests/helper/__init__.py b/dpctl/tests/helper/__init__.py new file mode 100644 index 0000000000..672505c384 --- /dev/null +++ b/dpctl/tests/helper/__init__.py @@ -0,0 +1,32 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper module for dpctl/tests +""" + +from ._helper import ( + create_invalid_capsule, + has_cpu, + has_gpu, + has_sycl_platforms, +) + +__all__ = [ + "create_invalid_capsule", + "has_cpu", + "has_gpu", + "has_sycl_platforms", +] diff --git a/dpctl/tests/_helper.py b/dpctl/tests/helper/_helper.py similarity index 100% rename from dpctl/tests/_helper.py rename to dpctl/tests/helper/_helper.py diff --git a/dpctl/tests/test_service.py b/dpctl/tests/test_service.py index 754ba2b09d..aadf320704 100644 --- a/dpctl/tests/test_service.py +++ b/dpctl/tests/test_service.py @@ -23,6 +23,9 @@ import os import os.path import re +import sys + +import pytest import dpctl @@ -67,6 +70,11 @@ def test_get_include(): def test_get_dpcppversion(): + """Intent of this test is to verify that libraries from dpcpp_cpp_rt + conda package used at run-time are not from an older oneAPI. Since these + libraries currently do not report the version, this test was using + a proxy (version of Intel(R) Math Kernel Library). + """ incl_dir = dpctl.get_include() libs = glob.glob(os.path.join(incl_dir, "..", "*DPCTLSyclInterface*")) libs = sorted(libs) @@ -80,13 +88,19 @@ def test_get_dpcppversion(): dpcpp_ver = dpcpp_ver.decode("utf-8") mkl_ver = _get_mkl_version_if_present() if mkl_ver is not None: - assert mkl_ver >= dpcpp_ver + if not mkl_ver >= dpcpp_ver: + pytest.xfail( + reason="Flaky test: Investigate Math Kernel Library " + f"library version {mkl_ver} being older than " + f"DPC++ version {dpcpp_ver} used to build dpctl" + ) def test___version__(): dpctl_ver = getattr(dpctl, "__version__", None) assert type(dpctl_ver) is str assert "unknown" not in dpctl_ver + assert "untagged" not in dpctl_ver # Reg expr from PEP-440, relaxed to allow for semantic variant # 0.9.0dev0 allowed, vs. PEP-440 compliant 0.9.0.dev0 reg_expr = ( @@ -95,3 +109,47 @@ def test___version__(): r"0|[1-9][0-9]*))?(\+.*)?$" ) assert re.match(reg_expr, dpctl_ver) is not None + + +def test_dev_utils(): + import tempfile + + import dpctl._diagnostics as dd + + ctx_mngr = dd.syclinterface_diagnostics + + with ctx_mngr(): + dpctl.SyclDevice().parent_device + with ctx_mngr(verbosity="error"): + dpctl.SyclDevice().parent_device + with pytest.raises(ValueError): + with ctx_mngr(verbosity="blah"): + dpctl.SyclDevice().parent_device + with tempfile.TemporaryDirectory() as temp_dir: + with ctx_mngr(log_dir=temp_dir): + dpctl.SyclDevice().parent_device + with pytest.raises(ValueError): + with ctx_mngr(log_dir="/not_a_dir"): + dpctl.SyclDevice().parent_device + + +def test_syclinterface(): + install_dir = os.path.dirname(os.path.abspath(dpctl.__file__)) + paths = glob.glob(os.path.join(install_dir, "*DPCTLSyclInterface*")) + if "linux" in sys.platform: + assert len(paths) > 1 and any( + [os.path.islink(fn) for fn in paths] + ), "All library instances are hard links" + elif sys.platform in ["win32", "cygwin"]: + exts = [] + for fn in paths: + _, file_ext = os.path.splitext(fn) + exts.append(file_ext.lower()) + assert ( + ".lib" in exts + ), "Installation does not have DPCTLSyclInterface.lib" + assert ( + ".dll" in exts + ), "Installation does not have DPCTLSyclInterface.dll" + else: + raise RuntimeError("Unsupported system") diff --git a/dpctl/tests/test_sycl_context.py b/dpctl/tests/test_sycl_context.py index 9d09a6117c..7f0bc5d5ee 100644 --- a/dpctl/tests/test_sycl_context.py +++ b/dpctl/tests/test_sycl_context.py @@ -18,11 +18,10 @@ """ import pytest +from helper import create_invalid_capsule import dpctl -from ._helper import create_invalid_capsule - list_of_valid_filter_selectors = [ "opencl", "opencl:gpu", @@ -191,15 +190,15 @@ def test_context_repr(): assert type(ctx.__repr__()) is str -def test_cpython_api(): +def test_cpython_api_SyclContext_GetContextRef(): import ctypes import sys ctx = dpctl.SyclContext() mod = sys.modules[ctx.__class__.__module__] - # get capsule storign get_context_ref function ptr - ctx_ref_fn_cap = mod.__pyx_capi__["get_context_ref"] - # construct Python callable to invoke "get_context_ref" + # get capsule storign SyclContext_GetContextRef function ptr + ctx_ref_fn_cap = mod.__pyx_capi__["SyclContext_GetContextRef"] + # construct Python callable to invoke "SyclContext_GetContextRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -214,6 +213,28 @@ def test_cpython_api(): assert r1 == r2 +def test_cpython_api_SyclContext_Make(): + import ctypes + import sys + + ctx = dpctl.SyclContext() + mod = sys.modules[ctx.__class__.__module__] + # get capsule storign SyclContext_Make function ptr + make_ctx_fn_cap = mod.__pyx_capi__["SyclContext_Make"] + # construct Python callable to invoke "SyclContext_Make" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_ctx_fn_ptr = cap_ptr_fn( + make_ctx_fn_cap, b"struct PySyclContextObject *(DPCTLSyclContextRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_ctx_fn = callable_maker(make_ctx_fn_ptr) + + ctx2 = make_ctx_fn(ctx.addressof_ref()) + assert ctx == ctx2 + + def test_invalid_capsule(): cap = create_invalid_capsule() with pytest.raises(ValueError): diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 5b3f30c89d..6277504ef2 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -219,11 +219,11 @@ def check_has_aspect_usm_restricted_shared_allocations(device): pytest.fail("has_aspect_usm_restricted_shared_allocations call failed") -def check_has_aspect_usm_system_allocator(device): +def check_has_aspect_usm_system_allocations(device): try: - device.has_aspect_usm_system_allocator + device.has_aspect_usm_system_allocations except Exception: - pytest.fail("has_aspect_usm_system_allocator call failed") + pytest.fail("has_aspect_usm_system_allocations call failed") def check_is_accelerator(device): @@ -520,7 +520,7 @@ def check_repr(device): check_has_aspect_usm_host_allocations, check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, - check_has_aspect_usm_system_allocator, + check_has_aspect_usm_system_allocations, check_get_max_read_image_args, check_get_max_write_image_args, check_get_image_2d_max_width, @@ -679,7 +679,7 @@ def test_hashing_of_device(): "usm_device_allocations", "usm_host_allocations", "usm_shared_allocations", - "usm_system_allocator", + "usm_system_allocations", ] # SYCL 2020 spec aspects not presently @@ -729,3 +729,48 @@ def test_handle_no_device(): dpctl.select_device_with_aspects(["gpu", "cpu"]) with pytest.raises(ValueError): dpctl.select_device_with_aspects("cpu", excluded_aspects="cpu") + + +def test_cpython_api_SyclDevice_GetDeviceRef(): + import ctypes + import sys + + d = dpctl.SyclDevice() + mod = sys.modules[d.__class__.__module__] + # get capsule storing SyclDevice_GetDeviceRef function ptr + d_ref_fn_cap = mod.__pyx_capi__["SyclDevice_GetDeviceRef"] + # construct Python callable to invoke "SyclDevice_GetDeviceRef" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + d_ref_fn_ptr = cap_ptr_fn( + d_ref_fn_cap, b"DPCTLSyclDeviceRef (struct PySyclDeviceObject *)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object) + get_device_ref_fn = callable_maker(d_ref_fn_ptr) + + r2 = d.addressof_ref() + r1 = get_device_ref_fn(d) + assert r1 == r2 + + +def test_cpython_api_SyclDevice_Make(): + import ctypes + import sys + + d = dpctl.SyclDevice() + mod = sys.modules[d.__class__.__module__] + # get capsule storign SyclContext_Make function ptr + make_d_fn_cap = mod.__pyx_capi__["SyclDevice_Make"] + # construct Python callable to invoke "SyclDevice_Make" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_d_fn_ptr = cap_ptr_fn( + make_d_fn_cap, b"struct PySyclDeviceObject *(DPCTLSyclDeviceRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_d_fn = callable_maker(make_d_fn_ptr) + + d2 = make_d_fn(d.addressof_ref()) + assert d == d2 diff --git a/dpctl/tests/test_sycl_event.py b/dpctl/tests/test_sycl_event.py index 4e76534bd8..7691f00aef 100644 --- a/dpctl/tests/test_sycl_event.py +++ b/dpctl/tests/test_sycl_event.py @@ -19,14 +19,13 @@ import numpy as np import pytest +from helper import create_invalid_capsule, has_cpu import dpctl import dpctl.memory as dpctl_mem import dpctl.program as dpctl_prog from dpctl import event_status_type as esty -from ._helper import create_invalid_capsule, has_cpu - def produce_event(profiling=False): oclSrc = " \ @@ -203,7 +202,7 @@ def test_sycl_timer(): # device task m1.copy_from_device(m2) # host task - [x ** 2 for x in range(1024)] + [x**2 for x in range(1024)] host_dt, device_dt = timer.dt assert host_dt > device_dt q_no_profiling = dpctl.SyclQueue() @@ -235,15 +234,15 @@ def test_addressof_ref(): assert type(ref) is int -def test_cpython_api(): +def test_cpython_api_SyclEvent_GetEventRef(): import ctypes import sys ev = dpctl.SyclEvent() mod = sys.modules[ev.__class__.__module__] - # get capsule storign get_event_ref function ptr - ev_ref_fn_cap = mod.__pyx_capi__["get_event_ref"] - # construct Python callable to invoke "get_event_ref" + # get capsule storign SyclEvent_GetEventRef function ptr + ev_ref_fn_cap = mod.__pyx_capi__["SyclEvent_GetEventRef"] + # construct Python callable to invoke "SyclEvent_GetEventRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -256,3 +255,25 @@ def test_cpython_api(): r2 = ev.addressof_ref() r1 = get_event_ref_fn(ev) assert r1 == r2 + + +def test_cpython_api_SyclEvent_Make(): + import ctypes + import sys + + ev = dpctl.SyclEvent() + mod = sys.modules[ev.__class__.__module__] + # get capsule storing SyclEvent_Make function ptr + make_e_fn_cap = mod.__pyx_capi__["SyclEvent_Make"] + # construct Python callable to invoke "SyclDevice_Make" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_e_fn_ptr = cap_ptr_fn( + make_e_fn_cap, b"struct PySyclEventObject *(DPCTLSyclEventRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_e_fn = callable_maker(make_e_fn_ptr) + + ev2 = make_e_fn(ev.addressof_ref()) + assert type(ev) == type(ev2) diff --git a/dpctl/tests/test_sycl_kernel_submit.py b/dpctl/tests/test_sycl_kernel_submit.py index 3f3a03bb97..8a60294dd5 100644 --- a/dpctl/tests/test_sycl_kernel_submit.py +++ b/dpctl/tests/test_sycl_kernel_submit.py @@ -25,6 +25,7 @@ import dpctl import dpctl.memory as dpctl_mem import dpctl.program as dpctl_prog +import dpctl.tensor as dpt @pytest.mark.parametrize( @@ -107,4 +108,99 @@ def test_create_program_from_source(ctype_str, dtype, ctypes_ctor): ref_c = a * np.array(d, dtype=dtype) + b host_dt, device_dt = timer.dt assert type(host_dt) is float and type(device_dt) is float - assert np.allclose(c, ref_c), "Faled for {}, {}".formatg(r, lr) + assert np.allclose(c, ref_c), "Failed for {}, {}".formatg(r, lr) + + +def test_async_submit(): + try: + q = dpctl.SyclQueue("opencl") + except dpctl.SyclQueueCreationError: + pytest.skip("OpenCL queue could not be created") + oclSrc = ( + "kernel void kern1(global unsigned int *res, unsigned int mod) {" + " size_t index = get_global_id(0);" + " int ri = (index % mod);" + " res[index] = (ri * ri) % mod;" + "}" + " " + "kernel void kern2(global unsigned int *res, unsigned int mod) {" + " size_t index = get_global_id(0);" + " int ri = (index % mod);" + " int ri2 = (ri * ri) % mod;" + " res[index] = (ri2 * ri) % mod;" + "}" + " " + "kernel void kern3(" + " global unsigned int *res, global unsigned int *arg1, " + " global unsigned int *arg2)" + "{" + " size_t index = get_global_id(0);" + " res[index] = " + " (arg1[index] < arg2[index]) ? arg1[index] : arg2[index];" + "}" + ) + prog = dpctl_prog.create_program_from_source(q, oclSrc) + kern1Kernel = prog.get_sycl_kernel("kern1") + kern2Kernel = prog.get_sycl_kernel("kern2") + kern3Kernel = prog.get_sycl_kernel("kern3") + + assert isinstance(kern1Kernel, dpctl_prog.SyclKernel) + assert isinstance(kern2Kernel, dpctl_prog.SyclKernel) + assert isinstance(kern2Kernel, dpctl_prog.SyclKernel) + + n = 1024 * 1024 + X = dpt.empty((3, n), dtype="u4", usm_type="device", sycl_queue=q) + first_row = dpctl_mem.as_usm_memory(X[0]) + second_row = dpctl_mem.as_usm_memory(X[1]) + third_row = dpctl_mem.as_usm_memory(X[2]) + + e1 = q.submit( + kern1Kernel, + [ + first_row, + ctypes.c_uint(17), + ], + [ + n, + ], + ) + e2 = q.submit( + kern2Kernel, + [ + second_row, + ctypes.c_uint(27), + ], + [ + n, + ], + ) + e3 = q.submit( + kern3Kernel, + [third_row, first_row, second_row], + [ + n, + ], + None, + [e1, e2], + ) + status_complete = dpctl.event_status_type.complete + assert not all( + [ + e == status_complete + for e in ( + e1.execution_status, + e2.execution_status, + e3.execution_status, + ) + ] + ) + + e3.wait() + Xnp = dpt.asnumpy(X) + Xref = np.empty((3, n), dtype="u4") + for i in range(n): + Xref[0, i] = (i * i) % 17 + Xref[1, i] = (i * i * i) % 27 + Xref[2, i] = min(Xref[0, i], Xref[1, i]) + + assert np.array_equal(Xnp, Xref) diff --git a/dpctl/tests/test_sycl_platform.py b/dpctl/tests/test_sycl_platform.py index 1333588d5e..3bc230cdb9 100644 --- a/dpctl/tests/test_sycl_platform.py +++ b/dpctl/tests/test_sycl_platform.py @@ -18,11 +18,10 @@ """ import pytest +from helper import has_sycl_platforms import dpctl -from ._helper import has_sycl_platforms - list_of_valid_filter_selectors = [ "opencl", "opencl:gpu", diff --git a/dpctl/tests/test_sycl_queue.py b/dpctl/tests/test_sycl_queue.py index bf35d76ae2..6ee176a726 100644 --- a/dpctl/tests/test_sycl_queue.py +++ b/dpctl/tests/test_sycl_queue.py @@ -21,11 +21,10 @@ import sys import pytest +from helper import create_invalid_capsule import dpctl -from ._helper import create_invalid_capsule - list_of_standard_selectors = [ dpctl.select_accelerator_device, dpctl.select_cpu_device, @@ -212,11 +211,11 @@ def check_has_aspect_usm_restricted_shared_allocations(device): pytest.fail("has_aspect_usm_restricted_shared_allocations call failed") -def check_has_aspect_usm_system_allocator(device): +def check_has_aspect_usm_system_allocations(device): try: - device.has_aspect_usm_system_allocator + device.has_aspect_usm_system_allocations except Exception: - pytest.fail("has_aspect_usm_system_allocator call failed") + pytest.fail("has_aspect_usm_system_allocations call failed") def check_is_accelerator(device): @@ -274,7 +273,7 @@ def check_is_host(device): check_has_aspect_usm_host_allocations, check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, - check_has_aspect_usm_system_allocator, + check_has_aspect_usm_system_allocations, ] @@ -470,12 +469,12 @@ def test_queue_capsule(): assert q2 != [] # compare with other types -def test_cpython_api(): +def test_cpython_api_SyclQueue_GetQueueRef(): q = dpctl.SyclQueue() mod = sys.modules[q.__class__.__module__] - # get capsule storign get_context_ref function ptr - q_ref_fn_cap = mod.__pyx_capi__["get_queue_ref"] - # construct Python callable to invoke "get_queue_ref" + # get capsule storign SyclQueue_GetQueueRef function ptr + q_ref_fn_cap = mod.__pyx_capi__["SyclQueue_GetQueueRef"] + # construct Python callable to invoke "SyclQueue_GetQueueRef" cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] @@ -490,6 +489,26 @@ def test_cpython_api(): assert r1 == r2 +def test_cpython_api_SyclQueue_Make(): + q = dpctl.SyclQueue() + mod = sys.modules[q.__class__.__module__] + # get capsule storing SyclQueue_Make function ptr + make_SyclQueue_fn_cap = mod.__pyx_capi__["SyclQueue_Make"] + # construct Python callable to invoke "SyclQueue_Make" + cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer + cap_ptr_fn.restype = ctypes.c_void_p + cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + make_SyclQueue_fn_ptr = cap_ptr_fn( + make_SyclQueue_fn_cap, b"struct PySyclQueueObject *(DPCTLSyclQueueRef)" + ) + callable_maker = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p) + make_SyclQueue_fn = callable_maker(make_SyclQueue_fn_ptr) + + q2 = make_SyclQueue_fn(q.addressof_ref()) + assert q.sycl_device == q2.sycl_device + assert q.sycl_context == q2.sycl_context + + def test_constructor_many_arg(): with pytest.raises(TypeError): dpctl.SyclQueue(None, None, None, None) diff --git a/dpctl/tests/test_sycl_queue_manager.py b/dpctl/tests/test_sycl_queue_manager.py index 0073f7fcd7..0904c6912b 100644 --- a/dpctl/tests/test_sycl_queue_manager.py +++ b/dpctl/tests/test_sycl_queue_manager.py @@ -20,33 +20,40 @@ import contextlib import pytest +from helper import has_cpu, has_gpu, has_sycl_platforms import dpctl -from ._helper import has_cpu, has_gpu, has_sycl_platforms - - -@pytest.mark.skipif( +skip_no_platform = pytest.mark.skipif( not has_sycl_platforms(), reason="No SYCL platforms available" ) +skip_no_gpu = pytest.mark.skipif( + not has_gpu(), reason="No OpenCL GPU queues available" +) +skip_no_cpu = pytest.mark.skipif( + not has_cpu(), reason="No OpenCL CPU queues available" +) + + +@skip_no_platform def test_is_in_device_context_outside_device_ctxt(): assert not dpctl.is_in_device_context() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_gpu def test_is_in_device_context_inside_device_ctxt_gpu(): with dpctl.device_context("opencl:gpu:0"): assert dpctl.is_in_device_context() -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_cpu def test_is_in_device_context_inside_device_ctxt_cpu(): with dpctl.device_context("opencl:cpu:0"): assert dpctl.is_in_device_context() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_is_in_device_context_inside_nested_device_ctxt(): with dpctl.device_context("opencl:cpu:0"): with dpctl.device_context("opencl:gpu:0"): @@ -55,7 +62,7 @@ def test_is_in_device_context_inside_nested_device_ctxt(): assert not dpctl.is_in_device_context() -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_cpu def test_is_in_device_context_inside_nested_device_ctxt_cpu(): cpu = dpctl.SyclDevice("cpu") n = cpu.max_compute_units @@ -76,17 +83,13 @@ def test_is_in_device_context_inside_nested_device_ctxt_cpu(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_get_current_device_type_outside_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_platform +@skip_no_gpu def test_get_current_device_type_inside_device_ctxt(): assert dpctl.get_current_device_type() is not None @@ -96,8 +99,8 @@ def test_get_current_device_type_inside_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") +@skip_no_cpu +@skip_no_gpu def test_get_current_device_type_inside_nested_device_ctxt(): assert dpctl.get_current_device_type() is not None @@ -111,15 +114,13 @@ def test_get_current_device_type_inside_nested_device_ctxt(): assert dpctl.get_current_device_type() is not None -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_num_current_queues_outside_with_clause(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_num_current_queues_inside_with_clause(): with dpctl.device_context("opencl:cpu:0"): assert 1 == dpctl.get_num_activated_queues() @@ -128,8 +129,8 @@ def test_num_current_queues_inside_with_clause(): assert 0 == dpctl.get_num_activated_queues() -@pytest.mark.skipif(not has_gpu(), reason="No OpenCL GPU queues available") -@pytest.mark.skipif(not has_cpu(), reason="No OpenCL CPU queues available") +@skip_no_gpu +@skip_no_cpu def test_num_current_queues_inside_threads(): from threading import Thread @@ -146,9 +147,7 @@ def SessionThread(): Session2.start() -@pytest.mark.skipif( - not has_sycl_platforms(), reason="No SYCL platforms available" -) +@skip_no_platform def test_get_current_backend(): dpctl.get_current_backend() dpctl.get_current_device_type() diff --git a/dpctl/tests/test_sycl_queue_memcpy.py b/dpctl/tests/test_sycl_queue_memcpy.py index f5cddc87b8..db1831b78c 100644 --- a/dpctl/tests/test_sycl_queue_memcpy.py +++ b/dpctl/tests/test_sycl_queue_memcpy.py @@ -18,12 +18,11 @@ """ import pytest +from helper import has_sycl_platforms import dpctl import dpctl.memory -from ._helper import has_sycl_platforms - def _create_memory(): nbytes = 1024 diff --git a/dpctl/tests/test_sycl_usm.py b/dpctl/tests/test_sycl_usm.py index da19c37668..9e728fb68b 100644 --- a/dpctl/tests/test_sycl_usm.py +++ b/dpctl/tests/test_sycl_usm.py @@ -19,6 +19,7 @@ import numpy as np import pytest +from helper import has_cpu, has_gpu, has_sycl_platforms import dpctl from dpctl.memory import ( @@ -28,8 +29,6 @@ as_usm_memory, ) -from ._helper import has_cpu, has_gpu, has_sycl_platforms - class Dummy(MemoryUSMShared): """ @@ -59,6 +58,8 @@ def test_memory_create(memory_ctor): assert len(mobj) == nbytes assert mobj.size == nbytes assert mobj._context == queue.sycl_context + assert mobj._queue == queue + assert mobj.sycl_queue == queue assert type(repr(mobj)) is str assert type(bytes(mobj)) is bytes assert sys.getsizeof(mobj) > nbytes @@ -495,7 +496,7 @@ def test_with_constructor(memory_ctor): shape=(64,), strides=(1,), offset=0, - syclobj=buf._queue._get_capsule(), + syclobj=buf.sycl_queue._get_capsule(), ) check_view(v) # Use context capsule @@ -529,33 +530,41 @@ def test_cpython_api(memory_ctor): mobj = memory_ctor(1024) mod = sys.modules[mobj.__class__.__module__] # get capsules storing function pointers - mem_ptr_fn_cap = mod.__pyx_capi__["get_usm_pointer"] - mem_ctx_fn_cap = mod.__pyx_capi__["get_context"] - mem_nby_fn_cap = mod.__pyx_capi__["get_nbytes"] - # construct Python callable to invoke "get_usm_pointer" + mem_ptr_fn_cap = mod.__pyx_capi__["Memory_GetUsmPointer"] + mem_q_ref_fn_cap = mod.__pyx_capi__["Memory_GetQueueRef"] + mem_ctx_ref_fn_cap = mod.__pyx_capi__["Memory_GetContextRef"] + mem_nby_fn_cap = mod.__pyx_capi__["Memory_GetNumBytes"] + # construct Python callable to invoke functions cap_ptr_fn = ctypes.pythonapi.PyCapsule_GetPointer cap_ptr_fn.restype = ctypes.c_void_p cap_ptr_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] mem_ptr_fn_ptr = cap_ptr_fn( mem_ptr_fn_cap, b"DPCTLSyclUSMRef (struct Py_MemoryObject *)" ) - mem_ctx_fn_ptr = cap_ptr_fn( - mem_ctx_fn_cap, b"DPCTLSyclContextRef (struct Py_MemoryObject *)" + mem_ctx_ref_fn_ptr = cap_ptr_fn( + mem_ctx_ref_fn_cap, b"DPCTLSyclContextRef (struct Py_MemoryObject *)" + ) + mem_q_ref_fn_ptr = cap_ptr_fn( + mem_q_ref_fn_cap, b"DPCTLSyclQueueRef (struct Py_MemoryObject *)" ) mem_nby_fn_ptr = cap_ptr_fn( mem_nby_fn_cap, b"size_t (struct Py_MemoryObject *)" ) callable_maker = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object) get_ptr_fn = callable_maker(mem_ptr_fn_ptr) - get_ctx_fn = callable_maker(mem_ctx_fn_ptr) + get_ctx_ref_fn = callable_maker(mem_ctx_ref_fn_ptr) + get_q_ref_fn = callable_maker(mem_q_ref_fn_ptr) get_nby_fn = callable_maker(mem_nby_fn_ptr) capi_ptr = get_ptr_fn(mobj) direct_ptr = mobj._pointer assert capi_ptr == direct_ptr - capi_ctx_ref = get_ctx_fn(mobj) + capi_ctx_ref = get_ctx_ref_fn(mobj) direct_ctx_ref = mobj._context.addressof_ref() assert capi_ctx_ref == direct_ctx_ref + capi_q_ref = get_q_ref_fn(mobj) + direct_q_ref = mobj.sycl_queue.addressof_ref() + assert capi_q_ref == direct_q_ref capi_nbytes = get_nby_fn(mobj) direct_nbytes = mobj.nbytes assert capi_nbytes == direct_nbytes diff --git a/dpctl/tests/test_tensor_asarray.py b/dpctl/tests/test_tensor_asarray.py new file mode 100644 index 0000000000..f4e3d77bfb --- /dev/null +++ b/dpctl/tests/test_tensor_asarray.py @@ -0,0 +1,220 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest + +import dpctl +import dpctl.tensor as dpt + + +@pytest.mark.parametrize( + "src_usm_type, dst_usm_type", + [ + ("device", "shared"), + ("device", "host"), + ("shared", "device"), + ("shared", "host"), + ("host", "device"), + ("host", "shared"), + ], +) +def test_asarray_change_usm_type(src_usm_type, dst_usm_type): + d = dpctl.SyclDevice() + if d.is_host: + pytest.skip( + "Skip test of host device, which only " + "supports host USM allocations" + ) + X = dpt.empty(10, dtype="u1", usm_type=src_usm_type) + Y = dpt.asarray(X, usm_type=dst_usm_type) + assert X.shape == Y.shape + assert X.usm_type == src_usm_type + assert Y.usm_type == dst_usm_type + + with pytest.raises(ValueError): + # zero copy is not possible + dpt.asarray(X, usm_type=dst_usm_type, copy=False) + + Y = dpt.asarray(X, usm_type=dst_usm_type, sycl_queue=X.sycl_queue) + assert X.shape == Y.shape + assert Y.usm_type == dst_usm_type + + Y = dpt.asarray( + X, + usm_type=dst_usm_type, + sycl_queue=X.sycl_queue, + device=d.get_filter_string(), + ) + assert X.shape == Y.shape + assert Y.usm_type == dst_usm_type + + +def test_asarray_from_numpy(): + Xnp = np.arange(10) + Y = dpt.asarray(Xnp, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == Xnp.shape + assert Y.dtype == Xnp.dtype + # Fortan contiguous case + Xnp = np.array([[1, 2, 3], [4, 5, 6]], dtype="f4", order="F") + Y = dpt.asarray(Xnp, usm_type="shared") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == Xnp.shape + assert Y.dtype == Xnp.dtype + # general strided case + Xnp = np.array([[1, 2, 3], [4, 5, 6]], dtype="i8") + Y = dpt.asarray(Xnp[::-1, ::-1], usm_type="host") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == Xnp.shape + assert Y.dtype == Xnp.dtype + + +def test_asarray_from_sequence(): + X = [1, 2, 3] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + + X = [(1, 1), (2.0, 2.0 + 1.0j), range(4, 6), np.array([3, 4], dtype="c16")] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.ndim == 2 + assert Y.shape == (len(X), 2) + + X = [] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == (0,) + + X = [[], []] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.shape == (2, 0) + + X = [True, False] + Y = dpt.asarray(X, usm_type="device") + assert type(Y) is dpt.usm_ndarray + assert Y.dtype.kind == "b" + + +def test_asarray_from_object_with_suai(): + """Test that asarray can deal with opaque objects implementing SUAI""" + + class Dummy: + def __init__(self, obj, iface): + self.obj = obj + self.__sycl_usm_array_interface__ = iface + + X = dpt.empty((2, 3, 4), dtype="f4") + Y = dpt.asarray(Dummy(X, X.__sycl_usm_array_interface__)) + assert Y.shape == X.shape + assert X.usm_type == Y.usm_type + assert X.dtype == Y.dtype + assert X.sycl_device == Y.sycl_device + + +def test_asarray_input_validation(): + with pytest.raises(TypeError): + # copy keyword is not of right type + dpt.asarray([1], copy="invalid") + with pytest.raises(TypeError): + # order keyword is not valid + dpt.asarray([1], order=1) + with pytest.raises(TypeError): + # dtype is not valid + dpt.asarray([1], dtype="invalid") + with pytest.raises(ValueError): + # unexpected value of order + dpt.asarray([1], order="Z") + with pytest.raises(TypeError): + # usm_type is of wrong type + dpt.asarray([1], usm_type=dict()) + with pytest.raises(ValueError): + # usm_type has wrong value + dpt.asarray([1], usm_type="mistake") + with pytest.raises(TypeError): + # sycl_queue type is not right + dpt.asarray([1], sycl_queue=dpctl.SyclContext()) + with pytest.raises(ValueError): + # sequence is not rectangular + dpt.asarray([[1], 2]) + + +def test_asarray_input_validation2(): + d = dpctl.get_devices() + if len(d) < 2: + pytest.skip("Not enough SYCL devices available") + + d0, d1 = d[:2] + try: + q0 = dpctl.SyclQueue(d0) + except dpctl.SyclQueueCreationError: + pytest.skip(f"SyclQueue could not be created for {d0}") + try: + q1 = dpctl.SyclQueue(d1) + except dpctl.SyclQueueCreationError: + pytest.skip(f"SyclQueue could not be created for {d1}") + with pytest.raises(TypeError): + dpt.asarray([1, 2], sycl_queue=q0, device=q1) + + +def test_asarray_scalars(): + import ctypes + + Y = dpt.asarray(5) + assert Y.dtype == np.dtype(int) + Y = dpt.asarray(5.2) + assert Y.dtype == np.dtype(float) + Y = dpt.asarray(np.float32(2.3)) + assert Y.dtype == np.dtype(np.float32) + Y = dpt.asarray(1.0j) + assert Y.dtype == np.dtype(complex) + Y = dpt.asarray(ctypes.c_int(8)) + assert Y.dtype == np.dtype(ctypes.c_int) + + +def test_asarray_copy_false(): + try: + q = dpctl.SyclQueue() + except dpctl.SyclQueueCreationError: + pytest.skip("Could not create a queue") + X = dpt.from_numpy(np.random.randn(10, 4), usm_type="device", sycl_queue=q) + Y1 = dpt.asarray(X, copy=False, order="K") + assert Y1 is X + Y1c = dpt.asarray(X, copy=True, order="K") + assert not (Y1c is X) + Y2 = dpt.asarray(X, copy=False, order="C") + assert Y2 is X + Y3 = dpt.asarray(X, copy=False, order="A") + assert Y3 is X + with pytest.raises(ValueError): + Y1 = dpt.asarray(X, copy=False, order="F") + Xf = dpt.empty( + X.shape, + dtype=X.dtype, + usm_type="device", + sycl_queue=X.sycl_queue, + order="F", + ) + Xf[:] = X + Y4 = dpt.asarray(Xf, copy=False, order="K") + assert Y4 is Xf + Y5 = dpt.asarray(Xf, copy=False, order="F") + assert Y5 is Xf + Y6 = dpt.asarray(Xf, copy=False, order="A") + assert Y6 is Xf + with pytest.raises(ValueError): + dpt.asarray(Xf, copy=False, order="C") diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index b5fab57566..c1c39633d5 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -18,13 +18,12 @@ import numbers import numpy as np -import numpy.lib.stride_tricks as np_st import pytest import dpctl import dpctl.memory as dpm import dpctl.tensor as dpt -from dpctl.tensor._usmarray import Device +from dpctl.tensor import Device @pytest.mark.parametrize( @@ -59,6 +58,16 @@ def test_allocate_usm_ndarray(shape, usm_type): assert X.shape == X.__sycl_usm_array_interface__["shape"] +def test_usm_ndarray_flags(): + assert dpt.usm_ndarray((5,)).flags == 3 + assert dpt.usm_ndarray((5, 2)).flags == 1 + assert dpt.usm_ndarray((5, 2), order="F").flags == 2 + assert dpt.usm_ndarray((5, 1, 2), order="F").flags == 2 + assert dpt.usm_ndarray((5, 1, 2), strides=(2, 0, 1)).flags == 1 + assert dpt.usm_ndarray((5, 1, 2), strides=(1, 0, 5)).flags == 2 + assert dpt.usm_ndarray((5, 1, 1), strides=(1, 0, 1)).flags == 3 + + @pytest.mark.parametrize( "dtype", [ @@ -115,6 +124,7 @@ def test_properties(dt): assert isinstance(X.nbytes, numbers.Integral) assert isinstance(X.ndim, numbers.Integral) assert isinstance(X._pointer, numbers.Integral) + assert isinstance(X.device, Device) @pytest.mark.parametrize("func", [bool, float, int, complex]) @@ -188,49 +198,10 @@ def test_basic_slice(ind): assert S.dtype == X.dtype -def _from_numpy(np_ary, device=None, usm_type="shared"): - if type(np_ary) is np.ndarray: - if np_ary.flags["FORC"]: - x = np_ary - else: - x = np.ascontiguous(np_ary) - R = dpt.usm_ndarray( - np_ary.shape, - dtype=np_ary.dtype, - buffer=usm_type, - buffer_ctor_kwargs={ - "queue": Device.create_device(device).sycl_queue - }, - ) - R.usm_data.copy_from_host(x.reshape((-1)).view("|u1")) - return R - else: - raise ValueError("Expected numpy.ndarray, got {}".format(type(np_ary))) - - -def _to_numpy(usm_ary): - if type(usm_ary) is dpt.usm_ndarray: - usm_buf = usm_ary.usm_data - s = usm_buf.nbytes - host_buf = usm_buf.copy_to_host().view(usm_ary.dtype) - usm_ary_itemsize = usm_ary.itemsize - R_offset = ( - usm_ary.__sycl_usm_array_interface__["offset"] * usm_ary_itemsize - ) - R = np.ndarray((s,), dtype="u1", buffer=host_buf) - R = R[R_offset:].view(usm_ary.dtype) - R_strides = (usm_ary_itemsize * si for si in usm_ary.strides) - return np_st.as_strided(R, shape=usm_ary.shape, strides=R_strides) - else: - raise ValueError( - "Expected dpctl.tensor.usm_ndarray, got {}".format(type(usm_ary)) - ) - - def test_slice_constructor_1d(): Xh = np.arange(37, dtype="i4") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type="device") + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type="device") for ind in [ slice(1, None, 2), slice(0, None, 3), @@ -242,14 +213,14 @@ def test_slice_constructor_1d(): slice(None, None, -13), ]: assert np.array_equal( - _to_numpy(Xusm[ind]), Xh[ind] + dpt.asnumpy(Xusm[ind]), Xh[ind] ), "Failed for {}".format(ind) def test_slice_constructor_3d(): Xh = np.empty((37, 24, 35), dtype="i4") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type="device") + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type="device") for ind in [ slice(1, None, 2), slice(0, None, 3), @@ -262,7 +233,7 @@ def test_slice_constructor_3d(): (slice(None, None, -2), Ellipsis, None, 15), ]: assert np.array_equal( - _to_numpy(Xusm[ind]), Xh[ind] + dpt.to_numpy(Xusm[ind]), Xh[ind] ), "Failed for {}".format(ind) @@ -270,7 +241,7 @@ def test_slice_constructor_3d(): def test_slice_suai(usm_type): Xh = np.arange(0, 10, dtype="u1") default_device = dpctl.select_default_device() - Xusm = _from_numpy(Xh, device=default_device, usm_type=usm_type) + Xusm = dpt.from_numpy(Xh, device=default_device, usm_type=usm_type) for ind in [slice(2, 3, None), slice(5, 7, None), slice(3, 9, None)]: assert np.array_equal( dpm.as_usm_memory(Xusm[ind]).copy_to_host(), Xh[ind] @@ -382,7 +353,7 @@ def test_pyx_capi_get_data(): X = dpt.usm_ndarray(17)[1::2] get_data_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_data", + "UsmNDArray_GetData", b"char *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -395,7 +366,7 @@ def test_pyx_capi_get_shape(): X = dpt.usm_ndarray(17)[1::2] get_shape_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_shape", + "UsmNDArray_GetShape", b"Py_ssize_t *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -408,7 +379,7 @@ def test_pyx_capi_get_strides(): X = dpt.usm_ndarray(17)[1::2] get_strides_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_strides", + "UsmNDArray_GetStrides", b"Py_ssize_t *(struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -424,7 +395,7 @@ def test_pyx_capi_get_ndim(): X = dpt.usm_ndarray(17)[1::2] get_ndim_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_ndim", + "UsmNDArray_GetNDim", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -435,7 +406,7 @@ def test_pyx_capi_get_typenum(): X = dpt.usm_ndarray(17)[1::2] get_typenum_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_typenum", + "UsmNDArray_GetTypenum", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -448,7 +419,7 @@ def test_pyx_capi_get_flags(): X = dpt.usm_ndarray(17)[1::2] get_flags_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_flags", + "UsmNDArray_GetFlags", b"int (struct PyUSMArrayObject *)", fn_restype=ctypes.c_int, ) @@ -456,11 +427,24 @@ def test_pyx_capi_get_flags(): assert type(flags) is int and flags == X.flags +def test_pyx_capi_get_offset(): + X = dpt.usm_ndarray(17)[1::2] + get_offset_fn = _pyx_capi_fnptr_to_callable( + X, + "UsmNDArray_GetOffset", + b"Py_ssize_t (struct PyUSMArrayObject *)", + fn_restype=ctypes.c_longlong, + ) + offset = get_offset_fn(X) + assert type(offset) is int + assert offset == X.__sycl_usm_array_interface__["offset"] + + def test_pyx_capi_get_queue_ref(): X = dpt.usm_ndarray(17)[1::2] get_queue_ref_fn = _pyx_capi_fnptr_to_callable( X, - "usm_ndarray_get_queue_ref", + "UsmNDArray_GetQueueRef", b"DPCTLSyclQueueRef (struct PyUSMArrayObject *)", fn_restype=ctypes.c_void_p, ) @@ -562,7 +546,7 @@ def test_pyx_capi_check_constants(): def test_tofrom_numpy(shape, dtype, usm_type): q = dpctl.SyclQueue() Xnp = np.zeros(shape, dtype=dtype) - Xusm = dpt.from_numpy(Xnp, usm_type=usm_type, queue=q) + Xusm = dpt.from_numpy(Xnp, usm_type=usm_type, sycl_queue=q) Ynp = np.ones(shape, dtype=dtype) ind = (slice(None, None, None),) * Ynp.ndim Xusm[ind] = Ynp @@ -629,6 +613,8 @@ def test_setitem_same_dtype(dtype, src_usm_type, dst_usm_type): R2 = np.broadcast_to(Xnp[0], R1.shape) assert R1.shape == R2.shape assert np.allclose(R1, R2) + Zusm_empty = Zusm_1d[0:0] + Zusm_empty[Ellipsis] = Zusm_3d[0, 0, 0:0] @pytest.mark.parametrize( @@ -703,11 +689,10 @@ def relaxed_strides_equal(st1, st2, sh): 5, ) X = dpt.usm_ndarray(sh_s, dtype="d") - expected_flags = X.flags X.shape = sh_f assert X.shape == sh_f assert relaxed_strides_equal(X.strides, cc_strides(sh_f), sh_f) - assert X.flags == expected_flags + assert X.flags & 1, "reshaped array expected to be C-contiguous" sh_s = ( 2, @@ -743,6 +728,21 @@ def relaxed_strides_equal(st1, st2, sh): X = dpt.usm_ndarray((4, 4), dtype="d")[::2, ::2] with pytest.raises(AttributeError): X.shape = (4,) + X = dpt.usm_ndarray((0,), dtype="i4") + X.shape = (0,) + X.shape = ( + 2, + 0, + ) + X.shape = ( + 0, + 2, + ) + X.shape = ( + 1, + 0, + 1, + ) def test_len(): @@ -777,13 +777,13 @@ def test_to_device(): def test_astype(): - X = dpt.usm_ndarray((5, 5), "i4") + X = dpt.empty((5, 5), dtype="i4") X[:] = np.full((5, 5), 7, dtype="i4") Y = dpt.astype(X, "c16", order="C") assert np.allclose(dpt.to_numpy(Y), np.full((5, 5), 7, dtype="c16")) - Y = dpt.astype(X, "f2", order="K") - assert np.allclose(dpt.to_numpy(Y), np.full((5, 5), 7, dtype="f2")) - Y = dpt.astype(X, "i4", order="K", copy=False) + Y = dpt.astype(X[::2, ::-1], "f2", order="K") + assert np.allclose(dpt.to_numpy(Y), np.full(Y.shape, 7, dtype="f2")) + Y = dpt.astype(X[::2, ::-1], "i4", order="K", copy=False) assert Y.usm_data is X.usm_data @@ -842,6 +842,36 @@ def test_reshape(): W = dpt.reshape(Z, (-1,), order="C") assert W.shape == (Z.size,) + X = dpt.usm_ndarray((1,)) + Y = dpt.reshape(X, X.shape) + assert Y.flags == X.flags + + A = dpt.usm_ndarray((0,), "i4") + A1 = dpt.reshape(A, (0,)) + assert A1.shape == (0,) + A2 = dpt.reshape( + A, + ( + 2, + 0, + ), + ) + assert A2.shape == ( + 2, + 0, + ) + A3 = dpt.reshape(A, (0, 2)) + assert A3.shape == ( + 0, + 2, + ) + A4 = dpt.reshape(A, (1, 0, 2)) + assert A4.shape == ( + 1, + 0, + 2, + ) + def test_transpose(): n, m = 2, 3 diff --git a/dpctl/tests/test_usm_ndarray_dlpack.py b/dpctl/tests/test_usm_ndarray_dlpack.py new file mode 100644 index 0000000000..a40688965b --- /dev/null +++ b/dpctl/tests/test_usm_ndarray_dlpack.py @@ -0,0 +1,120 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ctypes + +import pytest + +import dpctl +import dpctl.tensor as dpt + +device_oneAPI = 14 # DLDeviceType.kDLOneAPI + +_usm_types_list = ["shared", "device", "host"] + + +@pytest.fixture(params=_usm_types_list) +def usm_type(request): + return request.param + + +_typestrs_list = [ + "b1", + "u1", + "i1", + "u2", + "i2", + "u4", + "i4", + "u8", + "i8", + "f2", + "f4", + "f8", + "c8", + "c16", +] + + +@pytest.fixture(params=_typestrs_list) +def typestr(request): + return request.param + + +def test_dlpack_device(usm_type): + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty((64,), dtype="u1", usm_type=usm_type, device=sycl_dev) + dev = X.__dlpack_device__() + assert type(dev) is tuple + assert len(dev) == 2 + assert dev[0] == device_oneAPI + assert sycl_dev == all_root_devices[dev[1]] + + +def test_dlpack_exporter(typestr, usm_type): + caps_fn = ctypes.pythonapi.PyCapsule_IsValid + caps_fn.restype = bool + caps_fn.argtypes = [ctypes.py_object, ctypes.c_char_p] + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty((64,), dtype=typestr, usm_type=usm_type, device=sycl_dev) + caps = X.__dlpack__() + assert caps_fn(caps, b"dltensor") + Y = X[::2] + caps2 = Y.__dlpack__() + assert caps_fn(caps2, b"dltensor") + + +@pytest.mark.parametrize("shape", [tuple(), (2,), (3, 0, 1), (2, 2, 2)]) +def test_from_dlpack(shape, typestr, usm_type): + all_root_devices = dpctl.get_devices() + for sycl_dev in all_root_devices: + X = dpt.empty(shape, dtype=typestr, usm_type=usm_type, device=sycl_dev) + Y = dpt.from_dlpack(X) + assert X.shape == Y.shape + assert X.dtype == Y.dtype or ( + str(X.dtype) == "bool" and str(Y.dtype) == "uint8" + ) + assert X.sycl_device == Y.sycl_device + assert X.usm_type == Y.usm_type + assert X._pointer == Y._pointer + if Y.ndim: + V = Y[::-1] + W = dpt.from_dlpack(V) + assert V.strides == W.strides + + +def test_from_dlpack_input_validation(): + vstr = dpt._dlpack.get_build_dlpack_version() + assert type(vstr) is str + with pytest.raises(TypeError): + dpt.from_dlpack(None) + + class DummyWithProperty: + @property + def __dlpack__(self): + return None + + with pytest.raises(TypeError): + dpt.from_dlpack(DummyWithProperty()) + + class DummyWithMethod: + def __dlpack__(self): + return None + + with pytest.raises(TypeError): + dpt.from_dlpack(DummyWithMethod()) diff --git a/dpctl/utils/CMakeLists.txt b/dpctl/utils/CMakeLists.txt new file mode 100644 index 0000000000..11b0930052 --- /dev/null +++ b/dpctl/utils/CMakeLists.txt @@ -0,0 +1,6 @@ + +file(GLOB _cython_sources *.pyx) +foreach(_cy_file ${_cython_sources}) + get_filename_component(_trgt ${_cy_file} NAME_WLE) + build_dpctl_ext(${_trgt} ${_cy_file} "dpctl/utils") +endforeach() diff --git a/dpctl/utils/__init__.py b/dpctl/utils/__init__.py index f45afa0fab..7c52406ac0 100644 --- a/dpctl/utils/__init__.py +++ b/dpctl/utils/__init__.py @@ -1,3 +1,23 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A collection of utility functions. +""" + from ._compute_follows_data import get_execution_queue __all__ = [ diff --git a/examples/cython/sycl_buffer/.gitignore b/examples/cython/sycl_buffer/.gitignore new file mode 100644 index 0000000000..201efb635e --- /dev/null +++ b/examples/cython/sycl_buffer/.gitignore @@ -0,0 +1,3 @@ +_buffer_example.cpp +*.cpython*.so +*~ diff --git a/examples/cython/sycl_buffer/_buffer_example.pyx b/examples/cython/sycl_buffer/_buffer_example.pyx index 2737dbcc72..2f442d9388 100644 --- a/examples/cython/sycl_buffer/_buffer_example.pyx +++ b/examples/cython/sycl_buffer/_buffer_example.pyx @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# distutils: language = c++ +# cython: language_level=3 + cimport numpy as cnp import numpy as np diff --git a/examples/cython/sycl_buffer/bench.py b/examples/cython/sycl_buffer/bench.py index 9fc493fd18..f8edc91995 100644 --- a/examples/cython/sycl_buffer/bench.py +++ b/examples/cython/sycl_buffer/bench.py @@ -21,7 +21,7 @@ import dpctl -X = np.full((10 ** 4, 4098), 1e-4, dtype="d") +X = np.full((10**4, 4098), 1e-4, dtype="d") # warm-up print("=" * 10 + " Executing warm-up " + "=" * 10) diff --git a/examples/cython/sycl_buffer/run.py b/examples/cython/sycl_buffer/run.py index a82a6671a0..a66ae60dea 100644 --- a/examples/cython/sycl_buffer/run.py +++ b/examples/cython/sycl_buffer/run.py @@ -23,18 +23,31 @@ print("Result computed by NumPy") print(X.sum(axis=0)) -print("Result computed by SYCL extension using default offloading target") -print(sb.columnwise_total(X)) +try: + res = sb.columnwise_total(X) + print("Result computed by SYCL extension using default offloading target") + print(res) +except dpctl.SyclQueueCreationError: + print( + "Could not create SyclQueue for default selected device. Nothing to do." + ) + exit(0) print("") # controlling where to offload -q = dpctl.SyclQueue("opencl:gpu") -print("Running on: ", q.sycl_device.name) -print(sb.columnwise_total(X, queue=q)) - -q = dpctl.SyclQueue("opencl:cpu") -print("Running on: ", q.sycl_device.name) -print(sb.columnwise_total(X, queue=q)) +try: + q = dpctl.SyclQueue("opencl:gpu") + print("Running on: ", q.sycl_device.name) + print(sb.columnwise_total(X, queue=q)) +except dpctl.SyclQueueCreationError: + print("Not running onf opencl:gpu, queue could not be created") + +try: + q = dpctl.SyclQueue("opencl:cpu") + print("Running on: ", q.sycl_device.name) + print(sb.columnwise_total(X, queue=q)) +except dpctl.SyclQueueCreationError: + print("Not running onf opencl:cpu, queue could not be created") diff --git a/examples/cython/sycl_buffer/setup.py b/examples/cython/sycl_buffer/setup.py index 188fdda605..3e2d98390f 100644 --- a/examples/cython/sycl_buffer/setup.py +++ b/examples/cython/sycl_buffer/setup.py @@ -14,74 +14,58 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import os.path +import sysconfig + +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="syclbuffer", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on host allocated NumPy array using + SYCL buffers by calling oneMKL functions. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="syclbuffer", + sources=[ + "_buffer_example.pyx", + "use_sycl_buffer.cpp", + ], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="syclbuffer", - sources=[ - join(pdir, "_buffer_example.pyx"), - join(wdir, "use_sycl_buffer.cpp"), - join(wdir, "use_sycl_buffer.h"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.cpp b/examples/cython/sycl_buffer/use_sycl_buffer.cpp index 6abae0e73b..ba85510d09 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.cpp +++ b/examples/cython/sycl_buffer/use_sycl_buffer.cpp @@ -28,7 +28,7 @@ //===----------------------------------------------------------------------===// #include "use_sycl_buffer.h" -#include "dpctl_sycl_types.h" +#include "dpctl_sycl_interface.h" #include #include @@ -123,13 +123,13 @@ int c_columnwise_total_no_mkl(DPCTLSyclQueueRef q_ref, sycl::nd_range<2>(global, local), [=](sycl::nd_item<2> it) { size_t i = it.get_global_id(0); size_t j = it.get_global_id(1); - double group_sum = sycl::ONEAPI::reduce( + double group_sum = sycl::reduce_over_group( it.get_group(), (i < n) ? mat_acc[it.get_global_id()] : 0.0, std::plus()); if (it.get_local_id(0) == 0) { - sycl::ONEAPI::atomic_ref< - double, sycl::ONEAPI::memory_order::relaxed, - sycl::ONEAPI::memory_scope::system, + sycl::ext::oneapi::atomic_ref< + double, sycl::ext::oneapi::memory_order::relaxed, + sycl::ext::oneapi::memory_scope::system, sycl::access::address_space::global_space>(ct_acc[j]) += group_sum; } diff --git a/examples/cython/sycl_buffer/use_sycl_buffer.h b/examples/cython/sycl_buffer/use_sycl_buffer.h index 66bfe756bc..d9ea64c993 100644 --- a/examples/cython/sycl_buffer/use_sycl_buffer.h +++ b/examples/cython/sycl_buffer/use_sycl_buffer.h @@ -1,4 +1,4 @@ -#include "dpctl_sycl_types.h" +#include "dpctl_sycl_interface.h" #include extern int c_columnwise_total(DPCTLSyclQueueRef q, diff --git a/examples/cython/sycl_direct_linkage/.gitignore b/examples/cython/sycl_direct_linkage/.gitignore new file mode 100644 index 0000000000..201efb635e --- /dev/null +++ b/examples/cython/sycl_direct_linkage/.gitignore @@ -0,0 +1,3 @@ +_buffer_example.cpp +*.cpython*.so +*~ diff --git a/examples/cython/sycl_direct_linkage/_buffer_example.pyx b/examples/cython/sycl_direct_linkage/_buffer_example.pyx index 49d6cbd95e..effff5117c 100644 --- a/examples/cython/sycl_direct_linkage/_buffer_example.pyx +++ b/examples/cython/sycl_direct_linkage/_buffer_example.pyx @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# distutils: language = c++ +# cython: language_level=3 + cimport numpy as cnp import numpy as np diff --git a/examples/cython/sycl_direct_linkage/bench.py b/examples/cython/sycl_direct_linkage/bench.py index fdff5589ce..bc713a23d7 100644 --- a/examples/cython/sycl_direct_linkage/bench.py +++ b/examples/cython/sycl_direct_linkage/bench.py @@ -19,7 +19,7 @@ import numpy as np import syclbuffer_naive as sb -X = np.full((10 ** 4, 4098), 1e-4, dtype="d") +X = np.full((10**4, 4098), 1e-4, dtype="d") # warm-up print("=" * 10 + " Executing warm-up " + "=" * 10) diff --git a/examples/cython/sycl_direct_linkage/setup.py b/examples/cython/sycl_direct_linkage/setup.py index fe71d28a0a..b60e358afa 100644 --- a/examples/cython/sycl_direct_linkage/setup.py +++ b/examples/cython/sycl_direct_linkage/setup.py @@ -14,74 +14,62 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import os.path +import sysconfig + +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="syclbuffer", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on host allocated NumPy array using + SYCL buffers by calling oneMKL functions. + + This extension create SYCL queue in the scope of the function call + incurring large performance overhead. See `sycl_buffer/` example, + where user-constructed `dpctl.SyclQueue` can be given by the user. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="syclbuffer_naive", + sources=[ + "_buffer_example.pyx", + "sycl_function.cpp", + ], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="syclbuffer_naive", - sources=[ - join(pdir, "_buffer_example.pyx"), - join(pdir, "sycl_function.cpp"), - join(pdir, "sycl_function.hpp"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/cython/usm_memory/.gitignore b/examples/cython/usm_memory/.gitignore new file mode 100644 index 0000000000..d423caa640 --- /dev/null +++ b/examples/cython/usm_memory/.gitignore @@ -0,0 +1,3 @@ +blackscholes.cpp +*~ +*.cpython*.so diff --git a/examples/cython/usm_memory/run.py b/examples/cython/usm_memory/run.py index 2b676288dd..fa71bba36c 100644 --- a/examples/cython/usm_memory/run.py +++ b/examples/cython/usm_memory/run.py @@ -61,62 +61,51 @@ def gen_option_params( np.allclose(Xgpu, X_ref, atol=1e-5), ) -n_opts = 3 * 10 ** 6 +n_opts = 3 * 10**6 # compute on CPU sycl device -cpu_q = dpctl.SyclQueue("opencl:cpu:0") -opts1 = gen_option_params( - n_opts, - 20.0, - 30.0, - 22.0, - 29.0, - 18.0, - 24.0, - 0.01, - 0.05, - 0.01, - 0.05, - "d", - queue=cpu_q, -) - -gpu_q = dpctl.SyclQueue("level_zero:gpu:0") -opts2 = gen_option_params( - n_opts, - 20.0, - 30.0, - 22.0, - 29.0, - 18.0, - 24.0, - 0.01, - 0.05, - 0.01, - 0.05, - "d", - queue=gpu_q, -) - -cpu_times = [] -gpu_times = [] -for _ in range(5): - - t0 = timeit.default_timer() - X1 = bs.black_scholes_price(opts1, queue=cpu_q) - t1 = timeit.default_timer() - cpu_times.append(t1 - t0) - - # compute on GPU sycl device - - t0 = timeit.default_timer() - X2 = bs.black_scholes_price(opts2, queue=gpu_q) - t1 = timeit.default_timer() - gpu_times.append(t1 - t0) - -print("Using : {}".format(cpu_q.sycl_device.name)) -print("Wall times : {}".format(cpu_times)) - -print("Using : {}".format(gpu_q.sycl_device.name)) -print("Wall times : {}".format(gpu_times)) +queues = [] +for filter_str in ["cpu", "gpu"]: + try: + q = dpctl.SyclQueue(filter_str) + queues.append(q) + except dpctl.SyclQueueCreationError: + continue + +if not queues: + print("No queues could not created, nothing to do.") + exit(0) + +opt_params_list = [] +for q in queues: + opt_params = gen_option_params( + n_opts, + 20.0, + 30.0, + 22.0, + 29.0, + 18.0, + 24.0, + 0.01, + 0.05, + 0.01, + 0.05, + "d", + queue=q, + ) + opt_params_list.append(opt_params) + +times_dict = dict() +for q, params in zip(queues, opt_params_list): + times_list = [] + for _ in range(5): + t0 = timeit.default_timer() + X1 = bs.black_scholes_price(params, queue=q) + t1 = timeit.default_timer() + times_list.append(t1 - t0) + times_dict[q.name] = times_list + +for dev_name, wall_times in times_dict.items(): + print("Using : {}".format(dev_name)) + print("Wall times : {}".format(wall_times)) diff --git a/examples/cython/usm_memory/setup.py b/examples/cython/usm_memory/setup.py index a2a507f25b..cf5e4ed122 100644 --- a/examples/cython/usm_memory/setup.py +++ b/examples/cython/usm_memory/setup.py @@ -14,74 +14,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import dirname, join - -from Cython.Build import cythonize - - -def configuration(parent_package="", top_path=None): - import numpy as np - from numpy.distutils.misc_util import Configuration - - import dpctl - - config = Configuration("", parent_package, top_path) - - oneapi_root = environ.get("ONEAPI_ROOT", None) - if not oneapi_root: - raise ValueError( - "ONEAPI_ROOT must be set, typical value is /opt/intel/oneapi" +import os.path +import sysconfig + +import numpy as np +from setuptools import Extension, setup + +import dpctl + +setup( + name="blackscholes_usm", + version="0.0.0", + description="An example of Cython extension calling SYCL routines", + long_description=""" + Example of using SYCL to work on usm allocations. + + See README.md for more details. + """, + license="Apache 2.0", + author="Intel Corporation", + url="https://github.com/IntelPython/dpctl", + ext_modules=[ + Extension( + name="blackscholes_usm", + sources=[ + "blackscholes.pyx", + "sycl_blackscholes.cpp", + ], + include_dirs=[ + ".", + np.get_include(), + dpctl.get_include(), + os.path.join(sysconfig.get_paths()["include"], ".."), + ], + libraries=["sycl"] + + [ + "mkl_sycl", + "mkl_intel_ilp64", + "mkl_tbb_thread", + "mkl_core", + "tbb", + "iomp5", + ], + runtime_library_dirs=[], + extra_compile_args=[ + "-Wall", + "-Wextra", + "-fsycl", + "-fsycl-unnamed-lambda", + ], + extra_link_args=["-fPIC"], + language="c++", ) - - mkl_info = { - "include_dirs": [join(oneapi_root, "mkl", "include")], - "library_dirs": [ - join(oneapi_root, "mkl", "lib"), - join(oneapi_root, "mkl", "lib", "intel64"), - ], - "libraries": [ - "mkl_sycl", - "mkl_intel_ilp64", - "mkl_tbb_thread", - "mkl_core", - "tbb", - "iomp5", - ], - } - - mkl_include_dirs = mkl_info.get("include_dirs") - mkl_library_dirs = mkl_info.get("library_dirs") - mkl_libraries = mkl_info.get("libraries") - - pdir = dirname(__file__) - wdir = join(pdir) - - eca = ["-Wall", "-Wextra", "-fsycl", "-fsycl-unnamed-lambda"] - - config.add_extension( - name="blackscholes_usm", - sources=[ - join(pdir, "blackscholes.pyx"), - join(wdir, "sycl_blackscholes.cpp"), - join(wdir, "sycl_blackscholes.hpp"), - ], - include_dirs=[wdir, np.get_include(), dpctl.get_include()] - + mkl_include_dirs, - libraries=["sycl"] + mkl_libraries, - runtime_library_dirs=mkl_library_dirs, - extra_compile_args=eca, # + ['-O0', '-g', '-ggdb'], - extra_link_args=["-fPIC"], - language="c++", - ) - - config.ext_modules = cythonize( - config.ext_modules, include_path=[pdir, wdir] - ) - return config - - -if __name__ == "__main__": - from numpy.distutils.core import setup - - setup(configuration=configuration) + ], +) diff --git a/examples/cython/usm_memory/sycl_blackscholes.cpp b/examples/cython/usm_memory/sycl_blackscholes.cpp index 67406345b6..8d6d0d6f85 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.cpp +++ b/examples/cython/usm_memory/sycl_blackscholes.cpp @@ -27,7 +27,7 @@ //===----------------------------------------------------------------------===// #include "sycl_blackscholes.hpp" -#include "dpctl_sycl_types.h" +#include "dpctl_sycl_interface.h" #include #include #include diff --git a/examples/cython/usm_memory/sycl_blackscholes.hpp b/examples/cython/usm_memory/sycl_blackscholes.hpp index 02181e2756..1c38ebef4b 100644 --- a/examples/cython/usm_memory/sycl_blackscholes.hpp +++ b/examples/cython/usm_memory/sycl_blackscholes.hpp @@ -24,7 +24,7 @@ /// //===----------------------------------------------------------------------===// -#include "dpctl_sycl_types.h" +#include "dpctl_sycl_interface.h" #include template diff --git a/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp b/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp index 7a4e846f6d..30216dac55 100644 --- a/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp +++ b/examples/pybind11/external_usm_allocation/_usm_alloc_example.cpp @@ -33,12 +33,7 @@ //===----------------------------------------------------------------------===// #include -// clang-format off -#include "dpctl_sycl_types.h" -#include "../_sycl_queue.h" -#include "../_sycl_queue_api.h" -// clang-format on - +#include "dpctl4pybind11.hpp" #include "pybind11/pybind11.h" #include "pybind11/stl.h" @@ -87,19 +82,9 @@ struct DMatrix vec_t vec_; }; -DMatrix create_matrix(py::object queue, size_t n, size_t m) +DMatrix create_matrix(sycl::queue &q, size_t n, size_t m) { - PyObject *queue_ptr = queue.ptr(); - if (PyObject_TypeCheck(queue_ptr, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_ptr)); - sycl::queue *q = reinterpret_cast(QRef); - - return DMatrix(*q, n, m); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } + return DMatrix(q, n, m); } py::dict construct_sua_iface(DMatrix &m) @@ -149,8 +134,8 @@ py::list tolist(DMatrix &m) PYBIND11_MODULE(external_usm_alloc, m) { - // Import the dpctl._sycl_queue extension - import_dpctl___sycl_queue(); + // Import the dpctl extensions + import_dpctl(); py::class_ dm(m, "DMatrix"); dm.def(py::init(&create_matrix), diff --git a/examples/pybind11/external_usm_allocation/example.py b/examples/pybind11/external_usm_allocation/example.py index 522f822a36..4455616146 100644 --- a/examples/pybind11/external_usm_allocation/example.py +++ b/examples/pybind11/external_usm_allocation/example.py @@ -22,7 +22,7 @@ import dpctl import dpctl.memory as dpm -q = dpctl.SyclQueue("gpu") +q = dpctl.SyclQueue() matr = eua.DMatrix(q, 5, 5) print(matr) diff --git a/examples/pybind11/use_dpctl_syclqueue/_example.cpp b/examples/pybind11/use_dpctl_syclqueue/_example.cpp new file mode 100644 index 0000000000..7b26a35858 --- /dev/null +++ b/examples/pybind11/use_dpctl_syclqueue/_example.cpp @@ -0,0 +1,105 @@ +//==- pybind11_example.cpp - Example of Pybind11 extension working with -===// +// dpctl Python objects. +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements Pybind11-generated extension exposing functions that +/// take dpctl Python objects, such as dpctl.SyclQueue, dpctl.SyclDevice as +/// arguments. +/// +//===----------------------------------------------------------------------===// + +#include "dpctl4pybind11.hpp" +#include +#include +#include +#include + +namespace py = pybind11; + +uint64_t get_device_global_mem_size(sycl::device &d) +{ + return d.get_info(); +} + +uint64_t get_device_local_mem_size(sycl::device &d) +{ + return d.get_info(); +} + +py::array_t +offloaded_array_mod(sycl::queue &q, + py::array_t array, + int64_t mod) +{ + py::buffer_info arg_pybuf = array.request(); + if (arg_pybuf.ndim != 1) { + throw std::runtime_error("Expecting a vector"); + } + if (mod <= 0) { + throw std::runtime_error("Modulus must be non-negative"); + } + + size_t n = arg_pybuf.size; + + auto res = py::array_t(n); + py::buffer_info res_pybuf = res.request(); + + int64_t *a = static_cast(arg_pybuf.ptr); + int64_t *r = static_cast(res_pybuf.ptr); + + { + const sycl::property_list props = { + sycl::property::buffer::use_host_ptr()}; + sycl::buffer a_buf(a, sycl::range<1>(n), props); + sycl::buffer r_buf(r, sycl::range<1>(n), props); + + q.submit([&](sycl::handler &cgh) { + sycl::accessor a_acc(a_buf, cgh, sycl::read_only); + sycl::accessor r_acc(r_buf, cgh, sycl::write_only, sycl::no_init); + + cgh.parallel_for(sycl::range<1>(n), [=](sycl::id<1> idx) { + r_acc[idx] = a_acc[idx] % mod; + }); + }).wait_and_throw(); + } + + return res; +} + +PYBIND11_MODULE(use_queue_device_ext, m) +{ + // Import the dpctl extensions + import_dpctl(); + m.def( + "get_max_compute_units", + [=](sycl::queue &q) -> size_t { + return q.get_device() + .get_info(); + }, + "Computes max_compute_units property of the device underlying given " + "dpctl.SyclQueue"); + m.def("get_device_global_mem_size", &get_device_global_mem_size, + "Computes amount of global memory of the given dpctl.SyclDevice"); + m.def("get_device_local_mem_size", &get_device_local_mem_size, + "Computes amount of local memory of the given dpctl.SyclDevice"); + m.def("offloaded_array_mod", &offloaded_array_mod, + "Compute offloaded modular reduction of integer-valued NumPy array"); +} diff --git a/examples/pybind11/use_dpctl_syclqueue/example.py b/examples/pybind11/use_dpctl_syclqueue/example.py index 4c53bfafe2..7189adecb8 100644 --- a/examples/pybind11/use_dpctl_syclqueue/example.py +++ b/examples/pybind11/use_dpctl_syclqueue/example.py @@ -17,7 +17,7 @@ # coding: utf-8 import numpy as np -import pybind11_example as eg +import use_queue_device_ext as eg import dpctl @@ -36,7 +36,7 @@ print("") print("Computing modular reduction using SYCL on a NumPy array") -X = np.random.randint(low=1, high=2 ** 16 - 1, size=10 ** 6, dtype=np.longlong) +X = np.random.randint(low=1, high=2**16 - 1, size=10**6, dtype=np.longlong) modulus_p = 347 Y = eg.offloaded_array_mod( diff --git a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp b/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp deleted file mode 100644 index b90697a4f8..0000000000 --- a/examples/pybind11/use_dpctl_syclqueue/pybind11_example.cpp +++ /dev/null @@ -1,159 +0,0 @@ -//==- pybind11_example.cpp - Example of Pybind11 extension working with -===// -// dpctl Python objects. -// -// Data Parallel Control (dpctl) -// -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// This file implements Pybind11-generated extension exposing functions that -/// take dpctl Python objects, such as dpctl.SyclQueue, dpctl.SyclDevice as -/// arguments. -/// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include - -// clang-format off -// Ordering of includes is important here. dpctl_sycl_types defines types -// used by dpctl's Python C-API headers. -#include "dpctl_sycl_types.h" -#include "../_sycl_queue.h" -#include "../_sycl_queue_api.h" -#include "../_sycl_device.h" -#include "../_sycl_device_api.h" -// clang-format on - -namespace py = pybind11; - -size_t get_max_compute_units(py::object queue) -{ - PyObject *queue_ptr = queue.ptr(); - if (PyObject_TypeCheck(queue_ptr, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_ptr)); - sycl::queue *q = reinterpret_cast(QRef); - - return q->get_device() - .get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } -} - -uint64_t get_device_global_mem_size(py::object device) -{ - PyObject *device_pycapi = device.ptr(); - if (PyObject_TypeCheck(device_pycapi, &PySyclDeviceType)) { - DPCTLSyclDeviceRef DRef = get_device_ref( - reinterpret_cast(device_pycapi)); - sycl::device *d_ptr = reinterpret_cast(DRef); - return d_ptr->get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclDevice as argument"); - } -} - -uint64_t get_device_local_mem_size(py::object device) -{ - PyObject *device_pycapi = device.ptr(); - if (PyObject_TypeCheck(device_pycapi, &PySyclDeviceType)) { - DPCTLSyclDeviceRef DRef = get_device_ref( - reinterpret_cast(device_pycapi)); - sycl::device *d_ptr = reinterpret_cast(DRef); - return d_ptr->get_info(); - } - else { - throw std::runtime_error("expected dpctl.SyclDevice as argument"); - } -} - -py::array_t -offloaded_array_mod(py::object queue, - py::array_t array, - int64_t mod) -{ - sycl::queue *q_ptr; - - PyObject *queue_pycapi = queue.ptr(); - if (PyObject_TypeCheck(queue_pycapi, &PySyclQueueType)) { - DPCTLSyclQueueRef QRef = - get_queue_ref(reinterpret_cast(queue_pycapi)); - q_ptr = reinterpret_cast(QRef); - } - else { - throw std::runtime_error("expected dpctl.SyclQueue as argument"); - } - - py::buffer_info arg_pybuf = array.request(); - if (arg_pybuf.ndim != 1) { - throw std::runtime_error("Expecting a vector"); - } - if (mod <= 0) { - throw std::runtime_error("Modulus must be non-negative"); - } - - size_t n = arg_pybuf.size; - - auto res = py::array_t(n); - py::buffer_info res_pybuf = res.request(); - - int64_t *a = static_cast(arg_pybuf.ptr); - int64_t *r = static_cast(res_pybuf.ptr); - - { - const sycl::property_list props = { - sycl::property::buffer::use_host_ptr()}; - sycl::buffer a_buf(a, sycl::range<1>(n), props); - sycl::buffer r_buf(r, sycl::range<1>(n), props); - - q_ptr - ->submit([&](sycl::handler &cgh) { - sycl::accessor a_acc(a_buf, cgh, sycl::read_only); - sycl::accessor r_acc(r_buf, cgh, sycl::write_only, - sycl::noinit); - - cgh.parallel_for(sycl::range<1>(n), [=](sycl::id<1> idx) { - r_acc[idx] = a_acc[idx] % mod; - }); - }) - .wait_and_throw(); - } - - return res; -} - -PYBIND11_MODULE(pybind11_example, m) -{ - // Import the dpctl._sycl_queue, dpctl._sycl_device extensions - import_dpctl___sycl_device(); - import_dpctl___sycl_queue(); - m.def("get_max_compute_units", &get_max_compute_units, - "Computes max_compute_units property of the device underlying given " - "dpctl.SyclQueue"); - m.def("get_device_global_mem_size", &get_device_global_mem_size, - "Computes amount of global memory of the given dpctl.SyclDevice"); - m.def("get_device_local_mem_size", &get_device_local_mem_size, - "Computes amount of local memory of the given dpctl.SyclDevice"); - m.def("offloaded_array_mod", &offloaded_array_mod, - "Compute offloaded modular reduction of integer-valued NumPy array"); -} diff --git a/examples/pybind11/use_dpctl_syclqueue/setup.py b/examples/pybind11/use_dpctl_syclqueue/setup.py index 4569c99029..34eeebe38c 100644 --- a/examples/pybind11/use_dpctl_syclqueue/setup.py +++ b/examples/pybind11/use_dpctl_syclqueue/setup.py @@ -21,8 +21,8 @@ exts = [ Pybind11Extension( - "pybind11_example", - ["./pybind11_example.cpp"], + "use_queue_device_ext", + ["./_example.cpp"], include_dirs=[dpctl.get_include()], extra_compile_args=["-fPIC"], extra_link_args=["-fPIC"], diff --git a/examples/python/_runner.py b/examples/python/_runner.py index b26865ec5e..552965e3f0 100644 --- a/examples/python/_runner.py +++ b/examples/python/_runner.py @@ -55,7 +55,7 @@ def run_examples(example_description, glbls_dict): print("Available examples:") print(", ".join(fns)) else: - print("No examples are availble.") + print("No examples are available.") exit(0) if args.run == "all": fns = [] diff --git a/examples/python/device_selection.py b/examples/python/device_selection.py index 0acafe6a17..2278183c94 100644 --- a/examples/python/device_selection.py +++ b/examples/python/device_selection.py @@ -20,17 +20,6 @@ import dpctl -def print_device(d): - "Display information about given device argument." - if type(d) is not dpctl.SyclDevice: - raise ValueError - print("Name: ", d.name) - print("Vendor: ", d.vendor) - print("Driver version: ", d.driver_version) - print("Backend: ", d.backend) - print("Max EU: ", d.max_compute_units) - - def create_default_device(): """ Create default SyclDevice using `cl::sycl::default_selector`. @@ -42,7 +31,7 @@ def create_default_device(): d1 = dpctl.SyclDevice() d2 = dpctl.select_default_device() assert d1 == d2 - print_device(d1) + d1.print_device_info() return d1 @@ -54,11 +43,13 @@ def create_gpu_device(): SYCL_DEVICE_FILTER, which determines SYCL devices seen by the SYCL runtime. """ - d1 = dpctl.SyclDevice("gpu") - d2 = dpctl.select_gpu_device() - assert d1 == d2 - print_device(d1) - return d1 + try: + d1 = dpctl.SyclDevice("gpu") + d2 = dpctl.select_gpu_device() + assert d1 == d2 + d1.print_device_info() + except ValueError: + print("A GPU device is not available on the system") def create_gpu_device_if_present(): @@ -94,12 +85,50 @@ def custom_select_device(): max_score = d.default_selector_score selected_dev = d if selected_dev: - print_device(selected_dev) + selected_dev.print_device_info() else: print("No device with half-precision support is available.") return selected_dev +def create_device_with_aspects(): + """ + Programmatically select a device based on specific set of aspects. + + Demonstrate the usage of :func:`dpctl.select_device_with_aspects()`. + """ + dev = dpctl.select_device_with_aspects( + required_aspects=["fp64", "usm_shared_allocations"] + ) + dev.print_device_info() + + +def list_devices(): + """Programmatically get a list of the available devices. + + The list can be filtered based on backend or device_type. + """ + print("Get a list of all devices:\n") + + for d in dpctl.get_devices(): + d.print_device_info() + print("=======================================\n") + + print("Get the list of only OpenCL devices:\n") + + for d in dpctl.get_devices(backend="opencl"): + d.print_device_info() + + print("=======================================\n") + + print("Get all OpenCL CPU devices:\n") + + for d in dpctl.get_devices(backend="opencl", device_type="cpu"): + d.print_device_info() + + print("=======================================\n") + + if __name__ == "__main__": import _runner as runner diff --git a/examples/python/dppy_kernel.py b/examples/python/dppy_kernel.py index cfbbc947f6..05e7dd6cfc 100644 --- a/examples/python/dppy_kernel.py +++ b/examples/python/dppy_kernel.py @@ -44,7 +44,14 @@ def dppy_gemm(a, b, c): b = np.array(np.random.random(X * X), dtype=np.float32).reshape(X, X) c = np.ones_like(a).reshape(X, X) -q = dpctl.SyclQueue("opencl:gpu", property="enable_profiling") +try: + q = dpctl.SyclQueue("opencl:gpu", property="enable_profiling") +except dpctl.SyclQueueCreationError: + print( + "Skipping the example, as dpctl.SyclQueue targeting " + "opencl:gpu device could not be created" + ) + exit(0) timer = SyclTimer(time_scale=1) with dpctl.device_context(q): with timer(q): diff --git a/examples/python/filter_selection.py b/examples/python/filter_selection.py new file mode 100644 index 0000000000..af6dc91fc0 --- /dev/null +++ b/examples/python/filter_selection.py @@ -0,0 +1,53 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Examples illustrating SYCL device selection using filter strings. +""" + +import dpctl + + +def select_using_filter(): + """ + Demonstrate the usage of a filter string to create a SyclDevice. + + """ + try: + d1 = dpctl.SyclDevice("cpu") + d1.print_device_info() + except ValueError: + print("A CPU type device is not available on the system") + + try: + d1 = dpctl.SyclDevice("opencl:cpu:0") + d1.print_device_info() + except ValueError: + print("An OpenCL CPU driver needs to be installed on the system") + + d1 = dpctl.SyclDevice("0") + d1.print_device_info() + + try: + d1 = dpctl.SyclDevice("gpu") + d1.print_device_info() + except ValueError: + print("A GPU type device is not available on the system") + + +if __name__ == "__main__": + import _runner as runner + + runner.run_examples("Filter selection examples for dpctl.", globals()) diff --git a/examples/python/usm_memory_allocation.py b/examples/python/usm_memory_allocation.py index 5a54d117d4..caa1d456ac 100644 --- a/examples/python/usm_memory_allocation.py +++ b/examples/python/usm_memory_allocation.py @@ -34,8 +34,8 @@ # allocate using given queue, # i.e. on the device and bound to the context stored in the queue -mdq = dpmem.MemoryUSMDevice(256, queue=mda._queue) +mdq = dpmem.MemoryUSMDevice(256, queue=mda.sycl_queue) # information about device associate with USM buffer print("Allocation performed on device:") -mda._queue.sycl_device.print_device_info() +mda.sycl_queue.print_device_info() diff --git a/examples/python/usm_memory_host_access.py b/examples/python/usm_memory_host_access.py index 984b4273b5..5fdd24ffc4 100644 --- a/examples/python/usm_memory_host_access.py +++ b/examples/python/usm_memory_host_access.py @@ -34,7 +34,7 @@ # populate buffer from host one byte at a type for i in range(len(ms)): ir = i % 256 - msv[i] = ir ** 2 % 256 + msv[i] = ir**2 % 256 mh = dpmem.MemoryUSMHost(64) mhv = memoryview(mh) diff --git a/dpctl-capi/.gitignore b/libsyclinterface/.gitignore similarity index 100% rename from dpctl-capi/.gitignore rename to libsyclinterface/.gitignore diff --git a/dpctl-capi/CMakeLists.txt b/libsyclinterface/CMakeLists.txt similarity index 74% rename from dpctl-capi/CMakeLists.txt rename to libsyclinterface/CMakeLists.txt index bb921b8603..e63c992acb 100644 --- a/dpctl-capi/CMakeLists.txt +++ b/libsyclinterface/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10 FATAL_ERROR) +cmake_minimum_required(VERSION 3.10...3.22 FATAL_ERROR) project( "libDPCTLSYCLInterface" @@ -6,7 +6,7 @@ project( ) # Load our CMake modules to search for DPCPP and Level Zero -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/") find_package(Git REQUIRED) option(DPCTL_DPCPP_HOME_DIR @@ -38,6 +38,11 @@ option(DPCTL_BUILD_CAPI_TESTS "Build dpctl C API google tests" OFF ) +# Option to turn on logging support for dpctl C API +option(DPCTL_ENABLE_GLOG + "Enable the Google logging module" + OFF +) # Minimum version requirement only when oneAPI dpcpp is used. if(DPCTL_DPCPP_FROM_ONEAPI) @@ -69,8 +74,8 @@ if(DPCTL_ENABLE_LO_PROGRAM_CREATION) endif() configure_file( - ${CMAKE_SOURCE_DIR}/include/Config/dpctl_config.h.in - ${CMAKE_SOURCE_DIR}/include/Config/dpctl_config.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config/dpctl_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config/dpctl_config.h ) # Set the C++ standard to C++17 @@ -84,6 +89,7 @@ if(WIN32) "-Wunused-function " "-Wuninitialized " "-Wmissing-declarations " + "-Wno-deprecated-declarations " ) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WARNING_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS}") @@ -125,7 +131,7 @@ elseif(UNIX) "-fsycl " ) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAGS}") - set(CMAKE_CXX_FLAGS "${CXXFLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXFLAGS}") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${CFLAGS} -ggdb3 -DDEBUG" ) @@ -157,17 +163,33 @@ add_library(DPCTLSyclInterface ) target_include_directories(DPCTLSyclInterface + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/include/Support + ${CMAKE_CURRENT_SOURCE_DIR}/include/Config PRIVATE - ${CMAKE_SOURCE_DIR}/include/ - ${CMAKE_SOURCE_DIR}/helper/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/helper/include/ ${IntelSycl_SYCL_INCLUDE_DIR} ) - target_link_libraries(DPCTLSyclInterface - PRIVATE ${IntelSycl_SYCL_LIBRARY} - PRIVATE ${IntelSycl_OPENCL_LIBRARY} + PRIVATE ${IntelSycl_SYCL_LIBRARY} + PRIVATE ${IntelSycl_OPENCL_LIBRARY} ) +if(DPCTL_ENABLE_GLOG) + find_package(glog REQUIRED) + + target_include_directories(DPCTLSyclInterface + PRIVATE + glog::glog + ) + target_compile_definitions(DPCTLSyclInterface PRIVATE ENABLE_GLOG) + target_link_libraries(DPCTLSyclInterface + PRIVATE glog::glog + ) +endif() + + include(GetProjectVersion) # the get_version function is defined in the GetProjectVersion module and # defines: VERSION, SEMVER, MAJOR, MINOR, PATCH. These variables are populated @@ -186,29 +208,44 @@ if(DPCTL_ENABLE_LO_PROGRAM_CREATION) ) endif() +# Install all headers + +file(GLOB MAIN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") +file(GLOB SUPPORT_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Support/*.h") +file(GLOB CONFIG_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/Config/*.h") + +set_target_properties(DPCTLSyclInterface + PROPERTIES PUBLIC_HEADER + "${MAIN_HEADERS}" +) + +if (SKBUILD) + set(_lib_destination dpctl) + set(_include_destination dpctl/include/syclinterface) +else() + set(_lib_destination ${CMAKE_INSTALL_PREFIX}/lib) + set(_include_destination ${CMAKE_INSTALL_PREFIX}/include) +endif() + install(TARGETS DPCTLSyclInterface LIBRARY - DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ + DESTINATION ${_lib_destination} + ARCHIVE + DESTINATION ${_lib_destination} + RUNTIME + DESTINATION ${_lib_destination} + PUBLIC_HEADER + DESTINATION ${_include_destination} +) +install( + FILES ${SUPPORT_HEADERS} + DESTINATION ${_include_destination}/Support +) +install( + FILES ${CONFIG_HEADERS} + DESTINATION ${_include_destination}/Config ) - -# Install all headers -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include) -endforeach() - -# Install all headers in include/Support -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/Support/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include/Support) -endforeach() - -# Install all headers in include/Config -file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/Config/*.h") -foreach(HEADER ${HEADERS}) - install(FILES "${HEADER}" DESTINATION include/Config) -endforeach() # Enable code coverage related settings if(DPCTL_GENERATE_COVERAGE) diff --git a/dpctl-capi/cmake/modules/FindIntelSycl.cmake b/libsyclinterface/cmake/modules/FindIntelSycl.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindIntelSycl.cmake rename to libsyclinterface/cmake/modules/FindIntelSycl.cmake diff --git a/dpctl-capi/cmake/modules/FindLLVMCov.cmake b/libsyclinterface/cmake/modules/FindLLVMCov.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLLVMCov.cmake rename to libsyclinterface/cmake/modules/FindLLVMCov.cmake diff --git a/dpctl-capi/cmake/modules/FindLLVMProfdata.cmake b/libsyclinterface/cmake/modules/FindLLVMProfdata.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLLVMProfdata.cmake rename to libsyclinterface/cmake/modules/FindLLVMProfdata.cmake diff --git a/dpctl-capi/cmake/modules/FindLcov.cmake b/libsyclinterface/cmake/modules/FindLcov.cmake similarity index 100% rename from dpctl-capi/cmake/modules/FindLcov.cmake rename to libsyclinterface/cmake/modules/FindLcov.cmake diff --git a/dpctl-capi/cmake/modules/GetLevelZeroHeaders.cmake b/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake similarity index 92% rename from dpctl-capi/cmake/modules/GetLevelZeroHeaders.cmake rename to libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake index 83bb8e61a5..1b1cca582e 100644 --- a/dpctl-capi/cmake/modules/GetLevelZeroHeaders.cmake +++ b/libsyclinterface/cmake/modules/GetLevelZeroHeaders.cmake @@ -32,7 +32,7 @@ function(get_level_zero_headers) COMMAND ${GIT_EXECUTABLE} fetch RESULT_VARIABLE result ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -65,7 +65,7 @@ function(get_level_zero_headers) RESULT_VARIABLE result OUTPUT_VARIABLE latest_tag ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -81,7 +81,7 @@ function(get_level_zero_headers) COMMAND ${GIT_EXECUTABLE} checkout ${latest_tag} RESULT_VARIABLE result ERROR_VARIABLE error - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level-zero + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/level-zero OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE ) @@ -95,7 +95,7 @@ function(get_level_zero_headers) # Populate the path to the headers find_path(LEVEL_ZERO_INCLUDE_DIR NAMES zet_api.h - PATHS ${CMAKE_CURRENT_BINARY_DIR}/level-zero/include + PATHS ${CMAKE_BINARY_DIR}/level-zero/include NO_DEFAULT_PATH NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_PATH diff --git a/dpctl-capi/cmake/modules/GetProjectVersion.cmake b/libsyclinterface/cmake/modules/GetProjectVersion.cmake similarity index 100% rename from dpctl-capi/cmake/modules/GetProjectVersion.cmake rename to libsyclinterface/cmake/modules/GetProjectVersion.cmake diff --git a/dpctl-capi/dbg_build.bat b/libsyclinterface/dbg_build.bat similarity index 100% rename from dpctl-capi/dbg_build.bat rename to libsyclinterface/dbg_build.bat diff --git a/dpctl-capi/dbg_build.sh b/libsyclinterface/dbg_build.sh similarity index 100% rename from dpctl-capi/dbg_build.sh rename to libsyclinterface/dbg_build.sh diff --git a/dpctl-capi/dbg_build_custom.sh b/libsyclinterface/dbg_build_custom.sh similarity index 100% rename from dpctl-capi/dbg_build_custom.sh rename to libsyclinterface/dbg_build_custom.sh diff --git a/dpctl-capi/helper/include/dpctl_dynamic_lib_helper.h b/libsyclinterface/helper/include/dpctl_dynamic_lib_helper.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_dynamic_lib_helper.h rename to libsyclinterface/helper/include/dpctl_dynamic_lib_helper.h diff --git a/dpctl-capi/helper/include/dpctl_async_error_handler.h b/libsyclinterface/helper/include/dpctl_error_handlers.h similarity index 73% rename from dpctl-capi/helper/include/dpctl_async_error_handler.h rename to libsyclinterface/helper/include/dpctl_error_handlers.h index f409f8646c..9bd93eec66 100644 --- a/dpctl-capi/helper/include/dpctl_async_error_handler.h +++ b/libsyclinterface/helper/include/dpctl_error_handlers.h @@ -44,3 +44,22 @@ class DPCTL_API DPCTL_AsyncErrorHandler void operator()(const cl::sycl::exception_list &exceptions); }; + +enum error_level : int +{ + none = 0, + error = 1, + warning = 2 +}; + +void error_handler(const std::exception &e, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type = error_level::error); + +void error_handler(const std::string &what, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type = error_level::warning); diff --git a/dpctl-capi/helper/include/dpctl_string_utils.hpp b/libsyclinterface/helper/include/dpctl_string_utils.hpp similarity index 93% rename from dpctl-capi/helper/include/dpctl_string_utils.hpp rename to libsyclinterface/helper/include/dpctl_string_utils.hpp index aa1a8a9535..de13003813 100644 --- a/dpctl-capi/helper/include/dpctl_string_utils.hpp +++ b/libsyclinterface/helper/include/dpctl_string_utils.hpp @@ -21,6 +21,7 @@ /// \file /// Helper function to convert a C++ string to a C string. //===----------------------------------------------------------------------===// +#include "dpctl_error_handlers.h" #include #include #include @@ -55,9 +56,8 @@ cstring_from_string(const std::string &str) // to be null-terminated and the copy function is asked to // copy enough characters to include that null-character. cstr[cstr_len - 1] = '\0'; - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return cstr; diff --git a/dpctl-capi/helper/include/dpctl_utils_helper.h b/libsyclinterface/helper/include/dpctl_utils_helper.h similarity index 99% rename from dpctl-capi/helper/include/dpctl_utils_helper.h rename to libsyclinterface/helper/include/dpctl_utils_helper.h index e789c73517..57a308f5f0 100644 --- a/dpctl-capi/helper/include/dpctl_utils_helper.h +++ b/libsyclinterface/helper/include/dpctl_utils_helper.h @@ -24,8 +24,8 @@ #pragma once -#include "../include/dpctl_sycl_enum_types.h" #include "Support/DllExport.h" +#include "dpctl_sycl_enum_types.h" #include /*! diff --git a/dpctl-capi/helper/include/dpctl_vector_macros.h b/libsyclinterface/helper/include/dpctl_vector_macros.h similarity index 100% rename from dpctl-capi/helper/include/dpctl_vector_macros.h rename to libsyclinterface/helper/include/dpctl_vector_macros.h diff --git a/libsyclinterface/helper/source/dpctl_error_handlers.cpp b/libsyclinterface/helper/source/dpctl_error_handlers.cpp new file mode 100644 index 0000000000..e6b8adeabf --- /dev/null +++ b/libsyclinterface/helper/source/dpctl_error_handlers.cpp @@ -0,0 +1,135 @@ +//===-- dpctl_async_error_handler.h - An async error handler -*-C++-*- ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// A functor to use for passing an error handler callback function to sycl +/// context and queue contructors. +//===----------------------------------------------------------------------===// + +#include "dpctl_error_handlers.h" +#include "dpctl_service.h" +#include +#include +#ifdef ENABLE_GLOG +#include +#endif + +void DPCTL_AsyncErrorHandler::operator()( + const cl::sycl::exception_list &exceptions) +{ + for (std::exception_ptr const &e : exceptions) { + try { + std::rethrow_exception(e); + } catch (sycl::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); + auto err_code = e.code().value(); + handler_(static_cast(err_code)); + } + } +} + +namespace +{ +int requested_verbosity_level(void) +{ + int requested_level = 0; + + const char *verbose = std::getenv("DPCTL_VERBOSITY"); + + if (verbose) { + if (!std::strncmp(verbose, "none", 4)) + requested_level = error_level::none; + else if (!std::strncmp(verbose, "error", 5)) + requested_level = error_level::error; + else if (!std::strncmp(verbose, "warning", 7)) + requested_level = error_level::warning; + } + + return requested_level; +} + +void output_message(std::string ss_str, error_level error_type) +{ +#ifdef ENABLE_GLOG + switch (error_type) { + case error_level::error: + LOG(ERROR) << "[ERR] " << ss_str; + break; + case error_level::warning: + LOG(WARNING) << "[WARN] " << ss_str; + break; + default: + LOG(FATAL) << "[FATAL] " << ss_str; + } +#else + switch (error_type) { + case error_level::error: + std::cerr << "[ERR] " << ss_str; + break; + case error_level::warning: + std::cerr << "[WARN] " << ss_str; + break; + default: + std::cerr << "[FATAL] " << ss_str; + } +#endif +} + +} // namespace + +void error_handler(const std::exception &e, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type) +{ + int requested_level = requested_verbosity_level(); + int error_level = static_cast(error_type); + + bool to_output = requested_level >= error_level; + + if (to_output) { + std::stringstream ss; + ss << e.what() << " in " << func_name << " at " << file_name << ":" + << line_num << std::endl; + + output_message(ss.str(), error_type); + } +} + +void error_handler(const std::string &what, + const char *file_name, + const char *func_name, + int line_num, + error_level error_type) +{ + int requested_level = requested_verbosity_level(); + int error_level = static_cast(error_type); + + bool to_output = requested_level >= error_level; + + if (to_output) { + std::stringstream ss; + ss << what << " in " << func_name << " at " << file_name << ":" + << line_num << std::endl; + + output_message(ss.str(), error_type); + } +} diff --git a/dpctl-capi/helper/source/dpctl_utils_helper.cpp b/libsyclinterface/helper/source/dpctl_utils_helper.cpp similarity index 92% rename from dpctl-capi/helper/source/dpctl_utils_helper.cpp rename to libsyclinterface/helper/source/dpctl_utils_helper.cpp index 6cb0d977f5..c0013ddb71 100644 --- a/dpctl-capi/helper/source/dpctl_utils_helper.cpp +++ b/libsyclinterface/helper/source/dpctl_utils_helper.cpp @@ -93,11 +93,13 @@ backend DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType BeTy) case DPCTLSyclBackendType::DPCTL_HOST: return backend::host; case DPCTLSyclBackendType::DPCTL_LEVEL_ZERO: - return backend::level_zero; + return backend::ext_oneapi_level_zero; case DPCTLSyclBackendType::DPCTL_OPENCL: return backend::opencl; + case DPCTLSyclBackendType::DPCTL_ALL_BACKENDS: + return backend::all; default: - throw runtime_error("Unsupported backend type", -1); + throw std::runtime_error("Unsupported backend type"); } } @@ -108,7 +110,7 @@ DPCTLSyclBackendType DPCTL_SyclBackendToDPCTLBackendType(backend B) return DPCTLSyclBackendType::DPCTL_CUDA; case backend::host: return DPCTLSyclBackendType::DPCTL_HOST; - case backend::level_zero: + case backend::ext_oneapi_level_zero: return DPCTLSyclBackendType::DPCTL_LEVEL_ZERO; case backend::opencl: return DPCTLSyclBackendType::DPCTL_OPENCL; @@ -135,7 +137,7 @@ info::device_type DPCTL_DPCTLDeviceTypeToSyclDeviceType(DPCTLSyclDeviceType DTy) case DPCTLSyclDeviceType::DPCTL_HOST_DEVICE: return info::device_type::host; default: - throw runtime_error("Unsupported device type", -1); + throw std::runtime_error("Unsupported device type"); } } @@ -219,11 +221,11 @@ std::string DPCTL_AspectToStr(aspect aspectTy) case aspect::usm_restricted_shared_allocations: ss << "usm_restricted_shared_allocations"; break; - case aspect::usm_system_allocator: - ss << "usm_system_allocator"; + case aspect::usm_system_allocations: + ss << "usm_system_allocations"; break; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } return ss.str(); } @@ -285,12 +287,12 @@ aspect DPCTL_StrToAspectType(const std::string &aspectTyStr) else if (aspectTyStr == "usm_restricted_shared_allocations") { aspectTy = aspect::usm_restricted_shared_allocations; } - else if (aspectTyStr == "usm_system_allocator") { - aspectTy = aspect::usm_system_allocator; + else if (aspectTyStr == "usm_system_allocations") { + aspectTy = aspect::usm_system_allocations; } else { // \todo handle the error - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } return aspectTy; } @@ -332,10 +334,10 @@ aspect DPCTL_DPCTLAspectTypeToSyclAspect(DPCTLSyclAspectType AspectTy) return aspect::usm_shared_allocations; case DPCTLSyclAspectType::usm_restricted_shared_allocations: return aspect::usm_restricted_shared_allocations; - case DPCTLSyclAspectType::usm_system_allocator: - return aspect::usm_system_allocator; + case DPCTLSyclAspectType::usm_system_allocations: + return aspect::usm_system_allocations; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } } @@ -376,10 +378,10 @@ DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(aspect Aspect) return DPCTLSyclAspectType::usm_shared_allocations; case aspect::usm_restricted_shared_allocations: return DPCTLSyclAspectType::usm_restricted_shared_allocations; - case aspect::usm_system_allocator: - return DPCTLSyclAspectType::usm_system_allocator; + case aspect::usm_system_allocations: + return DPCTLSyclAspectType::usm_system_allocations; default: - throw runtime_error("Unsupported aspect type", -1); + throw std::runtime_error("Unsupported aspect type"); } } @@ -402,7 +404,7 @@ info::partition_affinity_domain DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( case DPCTLPartitionAffinityDomainType::next_partitionable: return info::partition_affinity_domain::next_partitionable; default: - throw runtime_error("Unsupported partition_affinity_domain type", -1); + throw std::runtime_error("Unsupported partition_affinity_domain type"); } } @@ -425,7 +427,7 @@ DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( case info::partition_affinity_domain::next_partitionable: return DPCTLPartitionAffinityDomainType::next_partitionable; default: - throw runtime_error("Unsupported partition_affinity_domain type", -1); + throw std::runtime_error("Unsupported partition_affinity_domain type"); } } diff --git a/dpctl-capi/include/Config/.gitignore b/libsyclinterface/include/Config/.gitignore similarity index 100% rename from dpctl-capi/include/Config/.gitignore rename to libsyclinterface/include/Config/.gitignore diff --git a/dpctl-capi/include/Config/dpctl_config.h.in b/libsyclinterface/include/Config/dpctl_config.h.in similarity index 100% rename from dpctl-capi/include/Config/dpctl_config.h.in rename to libsyclinterface/include/Config/dpctl_config.h.in diff --git a/dpctl-capi/include/Support/CBindingWrapping.h b/libsyclinterface/include/Support/CBindingWrapping.h similarity index 100% rename from dpctl-capi/include/Support/CBindingWrapping.h rename to libsyclinterface/include/Support/CBindingWrapping.h diff --git a/dpctl-capi/include/Support/DllExport.h b/libsyclinterface/include/Support/DllExport.h similarity index 100% rename from dpctl-capi/include/Support/DllExport.h rename to libsyclinterface/include/Support/DllExport.h diff --git a/dpctl-capi/include/Support/ExternC.h b/libsyclinterface/include/Support/ExternC.h similarity index 100% rename from dpctl-capi/include/Support/ExternC.h rename to libsyclinterface/include/Support/ExternC.h diff --git a/dpctl-capi/include/Support/MemOwnershipAttrs.h b/libsyclinterface/include/Support/MemOwnershipAttrs.h similarity index 100% rename from dpctl-capi/include/Support/MemOwnershipAttrs.h rename to libsyclinterface/include/Support/MemOwnershipAttrs.h diff --git a/dpctl-capi/include/dpctl_data_types.h b/libsyclinterface/include/dpctl_data_types.h similarity index 100% rename from dpctl-capi/include/dpctl_data_types.h rename to libsyclinterface/include/dpctl_data_types.h diff --git a/dpctl-capi/include/dpctl_error_handler_type.h b/libsyclinterface/include/dpctl_error_handler_type.h similarity index 100% rename from dpctl-capi/include/dpctl_error_handler_type.h rename to libsyclinterface/include/dpctl_error_handler_type.h diff --git a/dpctl-capi/include/dpctl_service.h b/libsyclinterface/include/dpctl_service.h similarity index 75% rename from dpctl-capi/include/dpctl_service.h rename to libsyclinterface/include/dpctl_service.h index e66c1a1c47..d7f9cfb552 100644 --- a/dpctl-capi/include/dpctl_service.h +++ b/libsyclinterface/include/dpctl_service.h @@ -43,4 +43,22 @@ DPCTL_C_EXTERN_C_BEGIN DPCTL_API __dpctl_give const char *DPCTLService_GetDPCPPVersion(void); +/*! + * @brief Initialize logger if compiled to use logger, no-op otherwise. + * + * @param app_name C-string for application name reflected in the log. + * @paral log_dir C-string for directory where log files are placed. + * @ingroup Service + */ +DPCTL_API +void DPCTLService_InitLogger(const char *app_name, const char *log_dir); + +/*! + * @brief Finilize logger if enabled, no-op otherwise. + * + * @ingroup Service + */ +DPCTL_API +void DPCTLService_ShutdownLogger(void); + DPCTL_C_EXTERN_C_END diff --git a/dpctl-capi/include/dpctl_sycl_context_interface.h b/libsyclinterface/include/dpctl_sycl_context_interface.h similarity index 95% rename from dpctl-capi/include/dpctl_sycl_context_interface.h rename to libsyclinterface/include/dpctl_sycl_context_interface.h index f93125092d..77434c576a 100644 --- a/dpctl-capi/include/dpctl_sycl_context_interface.h +++ b/libsyclinterface/include/dpctl_sycl_context_interface.h @@ -46,7 +46,7 @@ DPCTL_C_EXTERN_C_BEGIN * optional async error handler and properties bit flags. * * @param DRef Opaque pointer to a SYCL device. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during context creation. Can be * NULL if no async_handler is needed. * @param properties An optional combination of bit flags to define @@ -58,7 +58,7 @@ DPCTL_C_EXTERN_C_BEGIN DPCTL_API __dpctl_give DPCTLSyclContextRef DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! @@ -67,7 +67,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, * * @param DVRef An opaque pointer to a std::vector of * DPCTLSyclDeviceRef opaque pointers. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during context creation. Can be * NULL if no async_handler is needed. * @param properties An optional combination of bit flags to define @@ -79,7 +79,7 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, DPCTL_API __dpctl_give DPCTLSyclContextRef DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! diff --git a/dpctl-capi/include/dpctl_sycl_device_interface.h b/libsyclinterface/include/dpctl_sycl_device_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_interface.h rename to libsyclinterface/include/dpctl_sycl_device_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_device_manager.h b/libsyclinterface/include/dpctl_sycl_device_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_manager.h rename to libsyclinterface/include/dpctl_sycl_device_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_device_selector_interface.h b/libsyclinterface/include/dpctl_sycl_device_selector_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_device_selector_interface.h rename to libsyclinterface/include/dpctl_sycl_device_selector_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_enum_types.h b/libsyclinterface/include/dpctl_sycl_enum_types.h similarity index 99% rename from dpctl-capi/include/dpctl_sycl_enum_types.h rename to libsyclinterface/include/dpctl_sycl_enum_types.h index 86f1855129..afb97905ef 100644 --- a/dpctl-capi/include/dpctl_sycl_enum_types.h +++ b/libsyclinterface/include/dpctl_sycl_enum_types.h @@ -119,7 +119,7 @@ typedef enum usm_host_allocations, usm_shared_allocations, usm_restricted_shared_allocations, - usm_system_allocator + usm_system_allocations } DPCTLSyclAspectType; /*! diff --git a/dpctl-capi/include/dpctl_sycl_event_interface.h b/libsyclinterface/include/dpctl_sycl_event_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_event_interface.h rename to libsyclinterface/include/dpctl_sycl_event_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_kernel_interface.h b/libsyclinterface/include/dpctl_sycl_kernel_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_kernel_interface.h rename to libsyclinterface/include/dpctl_sycl_kernel_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_platform_interface.h b/libsyclinterface/include/dpctl_sycl_platform_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_platform_interface.h rename to libsyclinterface/include/dpctl_sycl_platform_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_platform_manager.h b/libsyclinterface/include/dpctl_sycl_platform_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_platform_manager.h rename to libsyclinterface/include/dpctl_sycl_platform_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_program_interface.h b/libsyclinterface/include/dpctl_sycl_program_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_program_interface.h rename to libsyclinterface/include/dpctl_sycl_program_interface.h diff --git a/dpctl-capi/include/dpctl_sycl_queue_interface.h b/libsyclinterface/include/dpctl_sycl_queue_interface.h similarity index 98% rename from dpctl-capi/include/dpctl_sycl_queue_interface.h rename to libsyclinterface/include/dpctl_sycl_queue_interface.h index 9f46a83e50..228604785e 100644 --- a/dpctl-capi/include/dpctl_sycl_queue_interface.h +++ b/libsyclinterface/include/dpctl_sycl_queue_interface.h @@ -47,7 +47,7 @@ DPCTL_C_EXTERN_C_BEGIN * * @param CRef An opaque pointer to a sycl::context. * @param DRef An opaque pointer to a sycl::device - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during queue creation. Can be * NULL if no async_handler is needed. * @param properties A combination of bit flags using the values defined @@ -62,7 +62,7 @@ DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, __dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! @@ -86,7 +86,7 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, * function begaves the same way as the SYCL constructor. * * @param dRef An opaque pointer to a ``sycl::device``. - * @param error_handler A callback function that will be invoked by the + * @param handler A callback function that will be invoked by the * async_handler used during queue creation. Can be * NULL if no async_handler is needed. * @param properties A combination of bit flags using the values defined @@ -101,7 +101,7 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef dRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties); /*! diff --git a/dpctl-capi/include/dpctl_sycl_queue_manager.h b/libsyclinterface/include/dpctl_sycl_queue_manager.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_queue_manager.h rename to libsyclinterface/include/dpctl_sycl_queue_manager.h diff --git a/dpctl-capi/include/dpctl_sycl_types.h b/libsyclinterface/include/dpctl_sycl_types.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_types.h rename to libsyclinterface/include/dpctl_sycl_types.h diff --git a/dpctl-capi/include/dpctl_sycl_usm_interface.h b/libsyclinterface/include/dpctl_sycl_usm_interface.h similarity index 100% rename from dpctl-capi/include/dpctl_sycl_usm_interface.h rename to libsyclinterface/include/dpctl_sycl_usm_interface.h diff --git a/dpctl-capi/include/dpctl_utils.h b/libsyclinterface/include/dpctl_utils.h similarity index 100% rename from dpctl-capi/include/dpctl_utils.h rename to libsyclinterface/include/dpctl_utils.h diff --git a/dpctl-capi/include/dpctl_vector.h b/libsyclinterface/include/dpctl_vector.h similarity index 100% rename from dpctl-capi/include/dpctl_vector.h rename to libsyclinterface/include/dpctl_vector.h diff --git a/dpctl-capi/source/dpctl_service.cpp b/libsyclinterface/source/dpctl_service.cpp similarity index 56% rename from dpctl-capi/source/dpctl_service.cpp rename to libsyclinterface/source/dpctl_service.cpp index 3457980721..7e96b6119d 100644 --- a/dpctl-capi/source/dpctl_service.cpp +++ b/libsyclinterface/source/dpctl_service.cpp @@ -26,13 +26,54 @@ #include "dpctl_service.h" #include "Config/dpctl_config.h" -#include "../helper/include/dpctl_string_utils.hpp" +#include "dpctl_string_utils.hpp" #include #include #include +#ifdef ENABLE_GLOG +#include +#include +#endif __dpctl_give const char *DPCTLService_GetDPCPPVersion(void) { std::string version = DPCTL_DPCPP_VERSION; return dpctl::helper::cstring_from_string(version); } + +#ifdef ENABLE_GLOG + +void DPCTLService_InitLogger(const char *app_name, const char *log_dir) +{ + google::InitGoogleLogging(app_name); + google::InstallFailureSignalHandler(); + + if (log_dir) { + namespace fs = std::filesystem; + const fs::path path(log_dir); + std::error_code ec; + + if (fs::is_directory(path, ec)) { + google::EnableLogCleaner(0); + FLAGS_log_dir = log_dir; + } + } + else { + FLAGS_colorlogtostderr = true; + FLAGS_stderrthreshold = google::FATAL; + FLAGS_logtostderr = 1; + } +} + +void DPCTLService_ShutdownLogger(void) +{ + google::ShutdownGoogleLogging(); +} + +#else +void DPCTLService_InitLogger([[maybe_unused]] const char *app_name, + [[maybe_unused]] const char *log_dir){}; + +void DPCTLService_ShutdownLogger(void){}; + +#endif diff --git a/dpctl-capi/source/dpctl_sycl_context_interface.cpp b/libsyclinterface/source/dpctl_sycl_context_interface.cpp similarity index 70% rename from dpctl-capi/source/dpctl_sycl_context_interface.cpp rename to libsyclinterface/source/dpctl_sycl_context_interface.cpp index e26f6ed49c..157b122236 100644 --- a/dpctl-capi/source/dpctl_sycl_context_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_context_interface.cpp @@ -25,8 +25,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_context_interface.h" -#include "../helper/include/dpctl_async_error_handler.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include #include @@ -43,20 +43,21 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(std::vector, __dpctl_give DPCTLSyclContextRef DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int /**/) { DPCTLSyclContextRef CRef = nullptr; auto Device = unwrap(DRef); - if (!Device) - return CRef; + if (!Device) { + error_handler("Cannot create device from DPCTLSyclDeviceRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); + return nullptr; + } try { - CRef = - wrap(new context(*Device, DPCTL_AsyncErrorHandler(error_handler))); - } catch (const std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - } catch (const runtime_error &re) { - std::cerr << re.what() << '\n'; + CRef = wrap(new context(*Device, DPCTL_AsyncErrorHandler(handler))); + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return CRef; @@ -64,14 +65,18 @@ DPCTLContext_Create(__dpctl_keep const DPCTLSyclDeviceRef DRef, __dpctl_give DPCTLSyclContextRef DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int /**/) { DPCTLSyclContextRef CRef = nullptr; std::vector Devices; auto DeviceRefs = unwrap(DVRef); - if (!DeviceRefs) + if (!DeviceRefs) { + error_handler("Cannot create device reference from DPCTLDeviceVectorRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return CRef; + } Devices.reserve(DeviceRefs->size()); for (auto const &DRef : *DeviceRefs) { @@ -79,12 +84,9 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, } try { - CRef = - wrap(new context(Devices, DPCTL_AsyncErrorHandler(error_handler))); - } catch (const std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - } catch (const runtime_error &re) { - std::cerr << re.what() << '\n'; + CRef = wrap(new context(Devices, DPCTL_AsyncErrorHandler(handler))); + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return CRef; @@ -93,9 +95,11 @@ DPCTLContext_CreateFromDevices(__dpctl_keep const DPCTLDeviceVectorRef DVRef, bool DPCTLContext_AreEq(__dpctl_keep const DPCTLSyclContextRef CtxRef1, __dpctl_keep const DPCTLSyclContextRef CtxRef2) { - if (!(CtxRef1 && CtxRef2)) - // \todo handle error + if (!(CtxRef1 && CtxRef2)) { + error_handler("DPCTLSyclContextRefs are nullptr.", __FILE__, __func__, + __LINE__); return false; + } return (*unwrap(CtxRef1) == *unwrap(CtxRef2)); } @@ -104,15 +108,15 @@ DPCTLContext_Copy(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Cannot copy DPCTLSyclContextRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclContextRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedContext = new context(*Context); return wrap(CopiedContext); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -122,16 +126,17 @@ DPCTLContext_GetDevices(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Can not retrieve devices from DPCTLSyclContextRef as " - "input is a nullptr\n"; + error_handler("Cannot retrieve devices from DPCTLSyclContextRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } std::vector *DevicesVectorPtr = nullptr; try { DevicesVectorPtr = new std::vector(); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + delete DevicesVectorPtr; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -141,15 +146,9 @@ DPCTLContext_GetDevices(__dpctl_keep const DPCTLSyclContextRef CRef) DevicesVectorPtr->emplace_back(wrap(new device(Dev))); } return wrap(DevicesVectorPtr); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete DevicesVectorPtr; - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (const runtime_error &re) { - delete DevicesVectorPtr; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -158,8 +157,9 @@ size_t DPCTLContext_DeviceCount(__dpctl_keep const DPCTLSyclContextRef CRef) { auto Context = unwrap(CRef); if (!Context) { - std::cerr << "Can not retrieve devices from DPCTLSyclContextRef as " - "input is a nullptr\n"; + error_handler("Cannot retrieve devices from DPCTLSyclContextRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); return 0; } const auto Devices = Context->get_devices(); @@ -183,6 +183,10 @@ void DPCTLContext_Delete(__dpctl_take DPCTLSyclContextRef CtxRef) DPCTLSyclBackendType DPCTLContext_GetBackend(__dpctl_keep const DPCTLSyclContextRef CtxRef) { + if (!CtxRef) { + return DPCTL_UNKNOWN_BACKEND; + } + auto BE = unwrap(CtxRef)->get_platform().get_backend(); switch (BE) { @@ -190,7 +194,7 @@ DPCTLContext_GetBackend(__dpctl_keep const DPCTLSyclContextRef CtxRef) return DPCTL_HOST; case backend::opencl: return DPCTL_OPENCL; - case backend::level_zero: + case backend::ext_oneapi_level_zero: return DPCTL_LEVEL_ZERO; case backend::cuda: return DPCTL_CUDA; @@ -207,8 +211,7 @@ size_t DPCTLContext_Hash(__dpctl_keep const DPCTLSyclContextRef CtxRef) return hash_fn(*C); } else { - std::cerr << "Argument CtxRef is null" - << "/n"; + error_handler("Argument CtxRef is null.", __FILE__, __func__, __LINE__); return 0; } } diff --git a/dpctl-capi/source/dpctl_sycl_device_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_interface.cpp similarity index 74% rename from dpctl-capi/source/dpctl_sycl_device_interface.cpp rename to libsyclinterface/source/dpctl_sycl_device_interface.cpp index dbe42e764b..47109fd21c 100644 --- a/dpctl-capi/source/dpctl_sycl_device_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_interface.cpp @@ -25,10 +25,11 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_interface.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include "dpctl_sycl_device_manager.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include #include @@ -52,15 +53,15 @@ DPCTLDevice_Copy(__dpctl_keep const DPCTLSyclDeviceRef DRef) { auto Device = unwrap(DRef); if (!Device) { - std::cerr << "Cannot copy DPCTLSyclDeviceRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclDeviceRef as input is a nullptr", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedDevice = new device(*Device); return wrap(CopiedDevice); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -70,13 +71,8 @@ __dpctl_give DPCTLSyclDeviceRef DPCTLDevice_Create() try { auto Device = new device(); return wrap(Device); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -85,19 +81,17 @@ __dpctl_give DPCTLSyclDeviceRef DPCTLDevice_CreateFromSelector( __dpctl_keep const DPCTLSyclDeviceSelectorRef DSRef) { auto Selector = unwrap(DSRef); - if (!Selector) - // \todo : Log error + if (!Selector) { + error_handler("Cannot difine device selector for DPCTLSyclDeviceRef " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; + } try { auto Device = new device(*Selector); return wrap(Device); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -116,9 +110,8 @@ DPCTLDevice_GetDeviceType(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto SyclDTy = D->get_info(); DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType(SyclDTy); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return DTy; @@ -180,9 +173,8 @@ DPCTLDevice_GetMaxComputeUnits(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { nComputeUnits = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return nComputeUnits; @@ -196,9 +188,8 @@ DPCTLDevice_GetGlobalMemSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { GlobalMemSize = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return GlobalMemSize; @@ -211,9 +202,8 @@ uint64_t DPCTLDevice_GetLocalMemSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { LocalMemSize = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return LocalMemSize; @@ -228,9 +218,8 @@ DPCTLDevice_GetMaxWorkItemDims(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { maxWorkItemDims = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return maxWorkItemDims; @@ -248,12 +237,8 @@ DPCTLDevice_GetMaxWorkItemSizes(__dpctl_keep const DPCTLSyclDeviceRef DRef) for (auto i = 0ul; i < 3; ++i) { sizes[i] = id_sizes[i]; } - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return sizes; @@ -267,9 +252,8 @@ DPCTLDevice_GetMaxWorkGroupSize(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { max_wg_size = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return max_wg_size; @@ -283,9 +267,8 @@ DPCTLDevice_GetMaxNumSubGroups(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { max_nsubgroups = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return max_nsubgroups; @@ -299,8 +282,8 @@ DPCTLDevice_GetPlatform(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { PRef = wrap(new platform(D->get_platform())); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return PRef; @@ -315,9 +298,8 @@ DPCTLDevice_GetName(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto name = D->get_info(); cstr_name = dpctl::helper::cstring_from_string(name); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_name; @@ -332,9 +314,8 @@ DPCTLDevice_GetVendor(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto vendor = D->get_info(); cstr_vendor = dpctl::helper::cstring_from_string(vendor); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_vendor; @@ -349,9 +330,8 @@ DPCTLDevice_GetDriverVersion(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto driver = D->get_info(); cstr_driver = dpctl::helper::cstring_from_string(driver); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_driver; @@ -364,9 +344,8 @@ bool DPCTLDevice_IsHostUnifiedMemory(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (D) { try { ret = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return ret; @@ -391,9 +370,8 @@ bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, if (D) { try { hasAspect = D->has(DPCTL_DPCTLAspectTypeToSyclAspect(AT)); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return hasAspect; @@ -407,8 +385,8 @@ bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, if (D) { \ try { \ result = D->get_info(); \ - } catch (runtime_error const &re) { \ - std::cerr << re.what() << '\n'; \ + } catch (std::exception const &e) { \ + error_handler(e, __FILE__, __func__, __LINE__); \ } \ } \ return result; \ @@ -431,9 +409,8 @@ bool DPCTLDevice_GetSubGroupIndependentForwardProgress( try { SubGroupProgress = D->get_info< info::device::sub_group_independent_forward_progress>(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return SubGroupProgress; @@ -448,9 +425,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthChar( try { vector_width_char = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_char; @@ -465,9 +441,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthShort( try { vector_width_short = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_short; @@ -482,9 +457,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthInt( try { vector_width_int = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_int; @@ -499,9 +473,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthLong( try { vector_width_long = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_long; @@ -516,9 +489,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthFloat( try { vector_width_float = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_float; @@ -533,9 +505,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthDouble( try { vector_width_double = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_double; @@ -550,9 +521,8 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthHalf( try { vector_width_half = D->get_info(); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return vector_width_half; @@ -566,12 +536,8 @@ DPCTLDevice_GetParentDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto parent_D = D->get_info(); return wrap(new device(parent_D)); - } catch (invalid_object_error const &ioe) { - // not a sub device - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -586,8 +552,8 @@ DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, std::vector *Devices = nullptr; if (DRef) { if (count == 0) { - std::cerr << "Can not create sub-devices with zero compute units" - << '\n'; + error_handler("Cannot create sub-devices with zero compute units", + __FILE__, __func__, __LINE__); return nullptr; } auto D = unwrap(DRef); @@ -598,18 +564,9 @@ DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (feature_not_supported const &fnse) { - delete Devices; - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -626,8 +583,8 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, vcounts.assign(counts, counts + ncounts); size_t min_elem = *std::min_element(vcounts.begin(), vcounts.end()); if (min_elem == 0) { - std::cerr << "Can not create sub-devices with zero compute units" - << '\n'; + error_handler("Cannot create sub-devices with zero compute units", + __FILE__, __func__, __LINE__); return nullptr; } if (DRef) { @@ -636,12 +593,8 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, try { subDevices = D->create_sub_devices< info::partition_property::partition_by_counts>(vcounts); - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -649,14 +602,9 @@ DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -679,18 +627,9 @@ __dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( for (const auto &sd : subDevices) { Devices->emplace_back(wrap(new device(sd))); } - } catch (std::bad_alloc const &ba) { - delete Devices; - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (feature_not_supported const &fnse) { - delete Devices; - std::cerr << fnse.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete Devices; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -705,9 +644,7 @@ size_t DPCTLDevice_Hash(__dpctl_keep const DPCTLSyclDeviceRef DRef) return hash_fn(*D); } else { - // todo: log error - std::cerr << "Argument DRef is null" - << "/n"; + error_handler("Argument DRef is null", __FILE__, __func__, __LINE__); return 0; } } diff --git a/dpctl-capi/source/dpctl_sycl_device_manager.cpp b/libsyclinterface/source/dpctl_sycl_device_manager.cpp similarity index 85% rename from dpctl-capi/source/dpctl_sycl_device_manager.cpp rename to libsyclinterface/source/dpctl_sycl_device_manager.cpp index 67446d6350..7a81b828b2 100644 --- a/dpctl-capi/source/dpctl_sycl_device_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_manager.cpp @@ -24,10 +24,11 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_device_manager.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include "dpctl_sycl_enum_types.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include #include @@ -125,18 +126,18 @@ struct DeviceCacheBuilder if (mRanker(D) < 0) continue; - // Per https://github.com/intel/llvm/blob/sycl/sycl/doc/ - // extensions/PlatformContext/PlatformContext.adoc - // sycl::queue(D) would create default platform context - // for capable compiler, sycl::context(D) otherwise - auto Q = queue(D); - auto Ctx = Q.get_context(); - auto entry = cache_l.emplace(D, Ctx); - - if (!entry.second) { - std::cerr << "Fatal Error during device cache " - "construction.\n"; - std::terminate(); + try { + // Per https://github.com/intel/llvm/blob/sycl/sycl/doc/ + // extensions/PlatformContext/PlatformContext.adoc + // sycl::queue(D) would create default platform context + // for capable compiler, sycl::context(D) otherwise + auto Q = queue(D); + auto Ctx = Q.get_context(); + cache_l.emplace(D, Ctx); + } catch (const std::exception &e) { + // Nothing is added to the cache_l by guarantees of + // emplace + error_handler(e, __FILE__, __func__, __LINE__); } } } @@ -160,9 +161,12 @@ DPCTLDeviceMgr_GetCachedContext(__dpctl_keep const DPCTLSyclDeviceRef DRef) DPCTLSyclContextRef CRef = nullptr; auto Device = unwrap(DRef); - if (!Device) + if (!Device) { + error_handler("Cannot create device from DPCTLSyclDeviceRef" + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return CRef; - + } auto &cache = DeviceCacheBuilder::getDeviceCache(); auto entry = cache.find(*Device); if (entry != cache.end()) { @@ -170,14 +174,15 @@ DPCTLDeviceMgr_GetCachedContext(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { ContextPtr = new context(entry->second); CRef = wrap(ContextPtr); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); delete ContextPtr; CRef = nullptr; } } else { - std::cerr << "No cached default context for device" << std::endl; + error_handler("No cached default context for device.", __FILE__, + __func__, __LINE__); } return CRef; } @@ -191,8 +196,9 @@ DPCTLDeviceMgr_GetDevices(int device_identifier) try { Devices = new std::vector(); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Devices; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -226,9 +232,8 @@ DPCTLDeviceMgr_GetDeviceInfoStr(__dpctl_keep const DPCTLSyclDeviceRef DRef) try { auto infostr = get_device_info_str(*D); cstr_info = dpctl::helper::cstring_from_string(infostr); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return cstr_info; @@ -280,7 +285,10 @@ size_t DPCTLDeviceMgr_GetNumDevices(int device_identifier) if (!device_identifier) return 0; + default_selector mRanker; for (const auto &entry : cache) { + if (mRanker(entry.first) < 0) + continue; auto Bty(DPCTL_SyclBackendToDPCTLBackendType( entry.first.get_platform().get_backend())); auto Dty(DPCTL_SyclDeviceTypeToDPCTLDeviceType( @@ -303,7 +311,8 @@ void DPCTLDeviceMgr_PrintDeviceInfo(__dpctl_keep const DPCTLSyclDeviceRef DRef) if (Device) std::cout << get_device_info_str(*Device); else - std::cout << "Device is not valid (NULL). Cannot print device info.\n"; + error_handler("Device is not valid (NULL). Cannot print device info.", + __FILE__, __func__, __LINE__); } int64_t DPCTLDeviceMgr_GetRelativeId(__dpctl_keep const DPCTLSyclDeviceRef DRef) diff --git a/dpctl-capi/source/dpctl_sycl_device_selector_interface.cpp b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp similarity index 74% rename from dpctl-capi/source/dpctl_sycl_device_selector_interface.cpp rename to libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp index 1193e11cee..fb87980aaa 100644 --- a/dpctl-capi/source/dpctl_sycl_device_selector_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_device_selector_interface.cpp @@ -25,6 +25,7 @@ #include "dpctl_sycl_device_selector_interface.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include /* SYCL headers */ using namespace cl::sycl; @@ -42,11 +43,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLAcceleratorSelector_Create() try { auto Selector = new accelerator_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -56,11 +54,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLDefaultSelector_Create() try { auto Selector = new default_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -70,11 +65,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLCPUSelector_Create() try { auto Selector = new cpu_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -90,11 +82,8 @@ DPCTLFilterSelector_Create(__dpctl_keep const char *filter_str) try { auto Selector = new filter_selector_t(filter_str); return wrap(Selector); - } catch (std::bad_alloc &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error &re) { - std::cerr << "Device: " << filter_str << " " << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -104,11 +93,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLGPUSelector_Create() try { auto Selector = new gpu_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -118,11 +104,8 @@ __dpctl_give DPCTLSyclDeviceSelectorRef DPCTLHostSelector_Create() try { auto Selector = new host_selector(); return wrap(Selector); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } diff --git a/dpctl-capi/source/dpctl_sycl_event_interface.cpp b/libsyclinterface/source/dpctl_sycl_event_interface.cpp similarity index 74% rename from dpctl-capi/source/dpctl_sycl_event_interface.cpp rename to libsyclinterface/source/dpctl_sycl_event_interface.cpp index 5d36ef3dee..406a873258 100644 --- a/dpctl-capi/source/dpctl_sycl_event_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_event_interface.cpp @@ -25,8 +25,9 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_event_interface.h" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_utils_helper.h" #include /* SYCL headers */ #include @@ -49,23 +50,23 @@ __dpctl_give DPCTLSyclEventRef DPCTLEvent_Create() try { auto E = new event(); ERef = wrap(E); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return ERef; } void DPCTLEvent_Wait(__dpctl_keep DPCTLSyclEventRef ERef) { - // \todo How to handle errors? E.g. when ERef is null or not a valid event. if (ERef) { auto SyclEvent = unwrap(ERef); if (SyclEvent) SyclEvent->wait(); } else { - std::cerr << "Cannot wait for the event. DPCTLSyclEventRef as input is " - "a nullptr\n"; + error_handler("Cannot wait for the event. DPCTLSyclEventRef as " + "input is a nullptr.", + __FILE__, __func__, __LINE__); } } @@ -77,9 +78,9 @@ void DPCTLEvent_WaitAndThrow(__dpctl_keep DPCTLSyclEventRef ERef) SyclEvent->wait_and_throw(); } else { - std::cerr << "Cannot wait_and_throw for the event. DPCTLSyclEventRef " - "as input is " - "a nullptr\n"; + error_handler("Cannot wait_and_throw for the event. DPCTLSyclEventRef " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); } } @@ -93,15 +94,15 @@ DPCTLEvent_Copy(__dpctl_keep DPCTLSyclEventRef ERef) { auto SyclEvent = unwrap(ERef); if (!SyclEvent) { - std::cerr << "Cannot copy DPCTLSyclEventRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclEventRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedSyclEvent = new event(*SyclEvent); return wrap(CopiedSyclEvent); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -114,7 +115,8 @@ DPCTLSyclBackendType DPCTLEvent_GetBackend(__dpctl_keep DPCTLSyclEventRef ERef) BTy = DPCTL_SyclBackendToDPCTLBackendType(E->get_backend()); } else { - std::cerr << "Backend cannot be looked up for a NULL event\n"; + error_handler("Backend cannot be looked up for a NULL event.", __FILE__, + __func__, __LINE__); } return BTy; } @@ -130,9 +132,8 @@ DPCTLEvent_GetCommandExecutionStatus(__dpctl_keep DPCTLSyclEventRef ERef) auto SyclESTy = E->get_info(); ESTy = DPCTL_SyclEventStatusToDPCTLEventStatusType(SyclESTy); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return ESTy; @@ -147,9 +148,8 @@ uint64_t DPCTLEvent_GetProfilingInfoSubmit(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoSubmit = E->get_profiling_info< sycl::info::event_profiling::command_submit>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoSubmit; @@ -164,9 +164,8 @@ uint64_t DPCTLEvent_GetProfilingInfoStart(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoStart = E->get_profiling_info< sycl::info::event_profiling::command_start>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoStart; @@ -181,9 +180,8 @@ uint64_t DPCTLEvent_GetProfilingInfoEnd(__dpctl_keep DPCTLSyclEventRef ERef) E->wait(); profilingInfoEnd = E->get_profiling_info< sycl::info::event_profiling::command_end>(); - } catch (invalid_object_error &e) { - // \todo log error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } return profilingInfoEnd; @@ -194,15 +192,15 @@ DPCTLEvent_GetWaitList(__dpctl_keep DPCTLSyclEventRef ERef) { auto E = unwrap(ERef); if (!E) { - std::cerr << "Cannot get wait list as input is a nullptr\n"; + error_handler("Cannot get wait list as input is a nullptr.", __FILE__, + __func__, __LINE__); return nullptr; } std::vector *EventsVectorPtr = nullptr; try { EventsVectorPtr = new std::vector(); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } try { @@ -212,15 +210,9 @@ DPCTLEvent_GetWaitList(__dpctl_keep DPCTLSyclEventRef ERef) EventsVectorPtr->emplace_back(wrap(new event(Ev))); } return wrap(EventsVectorPtr); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete EventsVectorPtr; - // \todo log error - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (const runtime_error &re) { - delete EventsVectorPtr; - // \todo log error - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } diff --git a/dpctl-capi/source/dpctl_sycl_kernel_interface.cpp b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp similarity index 82% rename from dpctl-capi/source/dpctl_sycl_kernel_interface.cpp rename to libsyclinterface/source/dpctl_sycl_kernel_interface.cpp index b25d73ada0..48d8ff919e 100644 --- a/dpctl-capi/source/dpctl_sycl_kernel_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_kernel_interface.cpp @@ -25,8 +25,9 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_kernel_interface.h" -#include "../helper/include/dpctl_string_utils.hpp" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" #include /* Sycl headers */ using namespace cl::sycl; @@ -42,7 +43,9 @@ __dpctl_give const char * DPCTLKernel_GetFunctionName(__dpctl_keep const DPCTLSyclKernelRef Kernel) { if (!Kernel) { - // \todo record error + error_handler("Cannot get the number of arguments " + "as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } @@ -56,7 +59,9 @@ DPCTLKernel_GetFunctionName(__dpctl_keep const DPCTLSyclKernelRef Kernel) size_t DPCTLKernel_GetNumArgs(__dpctl_keep const DPCTLSyclKernelRef Kernel) { if (!Kernel) { - // \todo record error + error_handler("Cannot get the number of arguments from " + "DPCTLSyclKernelRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return -1; } diff --git a/dpctl-capi/source/dpctl_sycl_platform_interface.cpp b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp similarity index 74% rename from dpctl-capi/source/dpctl_sycl_platform_interface.cpp rename to libsyclinterface/source/dpctl_sycl_platform_interface.cpp index 03004433cb..36c9bd16b7 100644 --- a/dpctl-capi/source/dpctl_sycl_platform_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_interface.cpp @@ -25,9 +25,10 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_interface.h" -#include "../helper/include/dpctl_string_utils.hpp" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" +#include "dpctl_string_utils.hpp" +#include "dpctl_utils_helper.h" #include #include #include @@ -50,15 +51,15 @@ DPCTLPlatform_Copy(__dpctl_keep const DPCTLSyclPlatformRef PRef) { auto Platform = unwrap(PRef); if (!Platform) { - std::cerr << "Cannot copy DPCTLSyclPlatformRef as input is a nullptr\n"; + error_handler("Cannot copy DPCTLSyclPlatformRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } try { auto CopiedPlatform = new platform(*Platform); return wrap(CopiedPlatform); - } catch (std::bad_alloc const &ba) { - // \todo log error - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -69,8 +70,8 @@ __dpctl_give DPCTLSyclPlatformRef DPCTLPlatform_Create() try { auto P = new platform(); PRef = wrap(P); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return PRef; } @@ -84,17 +85,15 @@ __dpctl_give DPCTLSyclPlatformRef DPCTLPlatform_CreateFromSelector( try { P = new platform(*DS); return wrap(P); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - return nullptr; - } catch (runtime_error const &re) { + } catch (std::exception const &e) { delete P; - std::cerr << re.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Device selector pointer cannot be NULL\n"; + error_handler("Device selector pointer cannot be NULL.", __FILE__, + __func__, __LINE__); } return nullptr; @@ -115,7 +114,8 @@ DPCTLPlatform_GetBackend(__dpctl_keep const DPCTLSyclPlatformRef PRef) BTy = DPCTL_SyclBackendToDPCTLBackendType(P->get_backend()); } else { - std::cerr << "Backend cannot be looked up for a NULL platform\n"; + error_handler("Backend cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); } return BTy; } @@ -128,14 +128,14 @@ DPCTLPlatform_GetName(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto name = P->get_info(); return dpctl::helper::cstring_from_string(name); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Name cannot be looked up for a NULL platform\n"; + error_handler("Name cannot be looked up for a NULL platform.", __FILE__, + __func__, __LINE__); return nullptr; } } @@ -148,14 +148,14 @@ DPCTLPlatform_GetVendor(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto vendor = P->get_info(); return dpctl::helper::cstring_from_string(vendor); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Vendor cannot be looked up for a NULL platform\n"; + error_handler("Vendor cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -168,14 +168,14 @@ DPCTLPlatform_GetVersion(__dpctl_keep const DPCTLSyclPlatformRef PRef) try { auto driver = P->get_info(); return dpctl::helper::cstring_from_string(driver); - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Driver version cannot be looked up for a NULL platform\n"; + error_handler("Driver version cannot be looked up for a NULL platform.", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -189,7 +189,8 @@ __dpctl_give DPCTLPlatformVectorRef DPCTLPlatform_GetPlatforms() try { Platforms = new std::vector(); Platforms->reserve(platforms.size()); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } diff --git a/dpctl-capi/source/dpctl_sycl_platform_manager.cpp b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp similarity index 91% rename from dpctl-capi/source/dpctl_sycl_platform_manager.cpp rename to libsyclinterface/source/dpctl_sycl_platform_manager.cpp index 11a4b65cf2..90dcf0ee71 100644 --- a/dpctl-capi/source/dpctl_sycl_platform_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_platform_manager.cpp @@ -25,9 +25,10 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_platform_manager.h" -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_platform_interface.h" +#include "dpctl_utils_helper.h" #include #include #include @@ -45,8 +46,9 @@ void platform_print_info_impl(const platform &p, size_t verbosity) std::stringstream ss; if (verbosity > 2) { - std::cerr << "Illegal verbosity level. Accepted values are 0, 1, or 2. " - "Defaulting to verbosity level 0.\n"; + error_handler("Illegal verbosity level. Accepted values are 0, 1, or 2." + "Defaulting to verbosity level 0.", + __FILE__, __func__, __LINE__); verbosity = 0; } @@ -112,6 +114,7 @@ void DPCTLPlatformMgr_PrintInfo(__dpctl_keep const DPCTLSyclPlatformRef PRef, platform_print_info_impl(*p, verbosity); } else { - std::cout << "Platform reference is NULL.\n"; + error_handler("Platform reference is NULL.", __FILE__, __func__, + __LINE__); } } diff --git a/dpctl-capi/source/dpctl_sycl_program_interface.cpp b/libsyclinterface/source/dpctl_sycl_program_interface.cpp similarity index 72% rename from dpctl-capi/source/dpctl_sycl_program_interface.cpp rename to libsyclinterface/source/dpctl_sycl_program_interface.cpp index 46b32c2de0..f5369f94ae 100644 --- a/dpctl-capi/source/dpctl_sycl_program_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_program_interface.cpp @@ -32,16 +32,20 @@ #include "dpctl_sycl_program_interface.h" #include "Config/dpctl_config.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include /* OpenCL headers */ #include /* Sycl headers */ #include +#include #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION -#include "../helper/include/dpctl_dynamic_lib_helper.h" -#include /* Level Zero headers */ +#include "dpctl_dynamic_lib_helper.h" // Note: include ze_api.h before level_zero.hpp. Make sure clang-format does // not reorder the includes. -#include +// clang-format off +#include "ze_api.h" /* Level Zero headers */ +#include "sycl/ext/oneapi/backend/level_zero.hpp" +// clang-format on #endif using namespace cl::sycl; @@ -84,10 +88,11 @@ createOpenCLInterOpProgram(const context &SyclCtx, auto CLCtx = get_native(SyclCtx); auto CLProgram = clCreateProgramWithIL(CLCtx, IL, length, &err); if (err) { - // \todo: record the error string and any other information. - std::cerr << "OpenCL program could not be created from the SPIR-V " - "binary. OpenCL Error " - << err << ".\n"; + std::stringstream ss; + ss << "OpenCL program could not be created from the SPIR-V " + "binary. OpenCL Error " + << err << "."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); return nullptr; } auto SyclDevices = SyclCtx.get_devices(); @@ -104,9 +109,9 @@ createOpenCLInterOpProgram(const context &SyclCtx, delete[] CLDevices; if (err) { - // \todo: record the error string and any other information. - std::cerr << "OpenCL program could not be built. OpenCL Error " << err - << ".\n"; + std::stringstream ss; + ss << "OpenCL program could not be built. OpenCL Error " << err << "."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); return nullptr; } @@ -114,9 +119,8 @@ createOpenCLInterOpProgram(const context &SyclCtx, try { auto SyclProgram = new program(SyclCtx, CLProgram); return wrap(SyclProgram); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -127,9 +131,9 @@ zeModuleCreateFT getZeModuleCreateFn() { static dpctl::DynamicLibHelper zeLib(zeLoaderName, libLoadFlags); if (!zeLib.opened()) { - // TODO: handle error - std::cerr << "The level zero loader dynamic library could not " - "be opened.\n"; + error_handler("The level zero loader dynamic library could not " + "be opened.", + __FILE__, __func__, __LINE__); return nullptr; } static auto stZeModuleCreateF = @@ -144,11 +148,11 @@ createLevelZeroInterOpProgram(const context &SyclCtx, size_t length, const char *CompileOpts) { - auto ZeCtx = get_native(SyclCtx); + auto ZeCtx = get_native(SyclCtx); auto SyclDevices = SyclCtx.get_devices(); if (SyclDevices.size() > 1) { - std::cerr << "Level zero program can be created for only one device.\n"; - // TODO: handle error + error_handler("Level zero program can be created for only one device.", + __FILE__, __func__, __LINE__); return nullptr; } @@ -159,38 +163,40 @@ createLevelZeroInterOpProgram(const context &SyclCtx, // Populate the Level Zero module descriptions ze_module_desc_t ZeModuleDesc = {}; + ZeModuleDesc.stype = ZE_STRUCTURE_TYPE_MODULE_DESC; ZeModuleDesc.format = ZE_MODULE_FORMAT_IL_SPIRV; ZeModuleDesc.inputSize = length; ZeModuleDesc.pInputModule = (uint8_t *)IL; ZeModuleDesc.pBuildFlags = CompileOpts; ZeModuleDesc.pConstants = &ZeSpecConstants; - auto ZeDevice = get_native(SyclDevices[0]); + auto ZeDevice = get_native(SyclDevices[0]); ze_module_handle_t ZeModule; auto stZeModuleCreateF = getZeModuleCreateFn(); if (!stZeModuleCreateF) { - std::cerr << "ZeModuleCreateFn is invalid.\n"; + error_handler("ZeModuleCreateFn is invalid.", __FILE__, __func__, + __LINE__); return nullptr; } auto ret = stZeModuleCreateF(ZeCtx, ZeDevice, &ZeModuleDesc, &ZeModule, nullptr); if (ret != ZE_RESULT_SUCCESS) { - // TODO: handle error - std::cerr << "ZeModule creation failed.\n"; + error_handler("ZeModule creation failed.", __FILE__, __func__, + __LINE__); return nullptr; } // Create the Sycl program from the ZeModule try { - auto ZeProgram = new program(sycl::level_zero::make_program( - SyclCtx, reinterpret_cast(ZeModule))); + auto ZeProgram = + new program(sycl::ext::oneapi::level_zero::make_program( + SyclCtx, reinterpret_cast(ZeModule))); return wrap(ZeProgram); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -207,9 +213,9 @@ DPCTLProgram_CreateFromSpirv(__dpctl_keep const DPCTLSyclContextRef CtxRef, DPCTLSyclProgramRef Pref = nullptr; context *SyclCtx = nullptr; if (!CtxRef) { - // \todo handle error - std::cerr << "Cannot create program from SPIR-V as the supplied SYCL " - "context is NULL.\n"; + error_handler("Cannot create program from SPIR-V as the supplied SYCL " + "context is NULL.", + __FILE__, __func__, __LINE__); return Pref; } SyclCtx = unwrap(CtxRef); @@ -219,7 +225,7 @@ DPCTLProgram_CreateFromSpirv(__dpctl_keep const DPCTLSyclContextRef CtxRef, case backend::opencl: Pref = createOpenCLInterOpProgram(*SyclCtx, IL, length, CompileOpts); break; - case backend::level_zero: + case backend::ext_oneapi_level_zero: #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION Pref = createLevelZeroInterOpProgram(*SyclCtx, IL, length, CompileOpts); #endif @@ -240,12 +246,12 @@ DPCTLProgram_CreateFromOCLSource(__dpctl_keep const DPCTLSyclContextRef Ctx, program *SyclProgram = nullptr; if (!Ctx) { - // \todo handle error + error_handler("Input Ctx is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } if (!Source) { - // \todo handle error message + error_handler("Input Source is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } @@ -264,29 +270,20 @@ DPCTLProgram_CreateFromOCLSource(__dpctl_keep const DPCTLSyclContextRef Ctx, try { SyclProgram->build_with_source(source, compileOpts); return wrap(SyclProgram); - } catch (compile_program_error &e) { - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { delete SyclProgram; - // \todo record error - return nullptr; - } catch (feature_not_supported &e) { - std::cerr << e.what() << '\n'; - delete SyclProgram; - // \todo record error - return nullptr; - } catch (runtime_error &e) { - std::cerr << e.what() << '\n'; - delete SyclProgram; - // \todo record error + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } break; - case backend::level_zero: - std::cerr << "CreateFromSource is not supported in Level Zero.\n"; + case backend::ext_oneapi_level_zero: + error_handler("CreateFromSource is not supported in Level Zero.", + __FILE__, __func__, __LINE__); delete SyclProgram; return nullptr; default: - std::cerr << "CreateFromSource is not supported in unknown backend.\n"; + error_handler("CreateFromSource is not supported in unknown backend.", + __FILE__, __func__, __LINE__); delete SyclProgram; return nullptr; } @@ -297,21 +294,21 @@ DPCTLProgram_GetKernel(__dpctl_keep DPCTLSyclProgramRef PRef, __dpctl_keep const char *KernelName) { if (!PRef) { - // \todo record error + error_handler("Input PRef is nullptr", __FILE__, __func__, __LINE__); return nullptr; } auto SyclProgram = unwrap(PRef); if (!KernelName) { - // \todo record error + error_handler("Input KernelName is nullptr", __FILE__, __func__, + __LINE__); return nullptr; } std::string name = KernelName; try { auto SyclKernel = new kernel(SyclProgram->get_kernel(name)); return wrap(SyclKernel); - } catch (invalid_object_error &e) { - // \todo record error - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -320,15 +317,20 @@ bool DPCTLProgram_HasKernel(__dpctl_keep DPCTLSyclProgramRef PRef, __dpctl_keep const char *KernelName) { if (!PRef) { - // \todo handle error + error_handler("Input PRef is nullptr", __FILE__, __func__, __LINE__); + return false; + } + if (!KernelName) { + error_handler("Input KernelName is nullptr", __FILE__, __func__, + __LINE__); return false; } auto SyclProgram = unwrap(PRef); try { return SyclProgram->has_kernel(KernelName); - } catch (invalid_object_error &e) { - std::cerr << e.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return false; } } diff --git a/dpctl-capi/source/dpctl_sycl_queue_interface.cpp b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp similarity index 78% rename from dpctl-capi/source/dpctl_sycl_queue_interface.cpp rename to libsyclinterface/source/dpctl_sycl_queue_interface.cpp index 46bbd0d10f..61630809d6 100644 --- a/dpctl-capi/source/dpctl_sycl_queue_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_interface.cpp @@ -25,8 +25,8 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_interface.h" -#include "../helper/include/dpctl_async_error_handler.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_context_interface.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_manager.h" @@ -108,9 +108,9 @@ bool set_kernel_arg(handler &cgh, cgh.set_arg(idx, Arg); break; default: - // \todo handle errors arg_set = false; - std::cerr << "Kernel argument could not be created.\n"; + error_handler("Kernel argument could not be created.", __FILE__, + __func__, __LINE__); break; } return arg_set; @@ -140,10 +140,10 @@ std::unique_ptr create_property_list(int properties) } if (_prop) { - // todo: log error - std::cerr << "Invalid queue property argument (" << std::hex - << properties << "), interpreted as (" << (properties ^ _prop) - << ")" << '\n'; + std::stringstream ss; + ss << "Invalid queue property argument (" << std::hex << properties + << "), interpreted as (" << (properties ^ _prop) << ")."; + error_handler(ss.str(), __FILE__, __func__, __LINE__); } return propList; } @@ -165,7 +165,7 @@ DPCTL_API __dpctl_give DPCTLSyclQueueRef DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, __dpctl_keep const DPCTLSyclDeviceRef DRef, - error_handler_callback *error_handler, + error_handler_callback *handler, int properties) { DPCTLSyclQueueRef q = nullptr; @@ -173,51 +173,45 @@ DPCTLQueue_Create(__dpctl_keep const DPCTLSyclContextRef CRef, auto ctx = unwrap(CRef); if (!(dev && ctx)) { - /* \todo handle error */ + error_handler("Cannot create queue from DPCTLSyclContextRef and " + "DPCTLSyclDeviceRef as input is a nullptr.", + __FILE__, __func__, __LINE__); return q; } auto propList = create_property_list(properties); - if (propList && error_handler) { + if (propList && handler) { try { - auto Queue = new queue( - *ctx, *dev, DPCTL_AsyncErrorHandler(error_handler), *propList); + auto Queue = new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler), + *propList); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else if (properties) { try { auto Queue = new queue(*ctx, *dev, *propList); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } - else if (error_handler) { + else if (handler) { try { auto Queue = - new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(error_handler)); + new queue(*ctx, *dev, DPCTL_AsyncErrorHandler(handler)); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else { try { auto Queue = new queue(*ctx, *dev); q = wrap(Queue); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } @@ -234,7 +228,8 @@ DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef, auto Device = unwrap(DRef); if (!Device) { - std::cerr << "Cannot create queue from NULL device reference.\n"; + error_handler("Cannot create queue from NULL device reference.", + __FILE__, __func__, __LINE__); return QRef; } // Check if a cached default context exists for the device. @@ -248,8 +243,8 @@ DPCTLQueue_CreateForDevice(__dpctl_keep const DPCTLSyclDeviceRef DRef, try { ContextPtr = new context(*Device); CRef = wrap(ContextPtr); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); delete ContextPtr; return QRef; } @@ -280,14 +275,14 @@ DPCTLQueue_Copy(__dpctl_keep const DPCTLSyclQueueRef QRef) try { auto CopiedQueue = new queue(*Queue); return wrap(CopiedQueue); - } catch (std::bad_alloc &ba) { - std::cerr << ba.what() << std::endl; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } else { - std::cerr << "Can not copy DPCTLSyclQueueRef as input is a nullptr" - << std::endl; + error_handler("Cannot copy DPCTLSyclQueueRef as input is a nullptr", + __FILE__, __func__, __LINE__); return nullptr; } } @@ -295,9 +290,11 @@ DPCTLQueue_Copy(__dpctl_keep const DPCTLSyclQueueRef QRef) bool DPCTLQueue_AreEq(__dpctl_keep const DPCTLSyclQueueRef QRef1, __dpctl_keep const DPCTLSyclQueueRef QRef2) { - if (!(QRef1 && QRef2)) - // \todo handle error + if (!(QRef1 && QRef2)) { + error_handler("DPCTLSyclQueueRefs are nullptr.", __FILE__, __func__, + __LINE__); return false; + } return (*unwrap(QRef1) == *unwrap(QRef2)); } @@ -308,9 +305,8 @@ DPCTLSyclBackendType DPCTLQueue_GetBackend(__dpctl_keep DPCTLSyclQueueRef QRef) try { auto C = Q->get_context(); return DPCTLContext_GetBackend(wrap(&C)); - } catch (runtime_error &re) { - std::cerr << re.what() << '\n'; - // store error message + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return DPCTL_UNKNOWN_BACKEND; } } @@ -327,12 +323,13 @@ DPCTLQueue_GetDevice(__dpctl_keep const DPCTLSyclQueueRef QRef) try { auto Device = new device(Q->get_device()); DRef = wrap(Device); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } } else { - std::cerr << "Could not get the device for this queue.\n"; + error_handler("Could not get the device for this queue.", __FILE__, + __func__, __LINE__); } return DRef; } @@ -345,7 +342,8 @@ DPCTLQueue_GetContext(__dpctl_keep const DPCTLSyclQueueRef QRef) if (Q) CRef = wrap(new context(Q->get_context())); else { - std::cerr << "Could not get the context for this queue.\n"; + error_handler("Could not get the context for this queue.", __FILE__, + __func__, __LINE__); } return CRef; } @@ -374,7 +372,6 @@ DPCTLQueue_SubmitRange(__dpctl_keep const DPCTLSyclKernelRef KRef, for (auto i = 0ul; i < NArgs; ++i) { // \todo add support for Sycl buffers - // \todo handle errors properly if (!set_kernel_arg(cgh, i, Args[i], ArgTypes[i])) exit(1); } @@ -390,17 +387,12 @@ DPCTLQueue_SubmitRange(__dpctl_keep const DPCTLSyclKernelRef KRef, *Kernel); break; default: - // \todo handle the error throw std::runtime_error("Range cannot be greater than three " "dimensions."); } }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -432,7 +424,6 @@ DPCTLQueue_SubmitNDRange(__dpctl_keep const DPCTLSyclKernelRef KRef, for (auto i = 0ul; i < NArgs; ++i) { // \todo add support for Sycl buffers - // \todo handle errors properly if (!set_kernel_arg(cgh, i, Args[i], ArgTypes[i])) exit(1); } @@ -452,17 +443,12 @@ DPCTLQueue_SubmitNDRange(__dpctl_keep const DPCTLSyclKernelRef KRef, *Kernel); break; default: - // \todo handle the error throw std::runtime_error("Range cannot be greater than three " "dimensions."); } }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } @@ -479,8 +465,7 @@ void DPCTLQueue_Wait(__dpctl_keep DPCTLSyclQueueRef QRef) SyclQueue->wait(); } else { - // todo: log error - std::cerr << "Argument QRef is NULL" << '\n'; + error_handler("Argument QRef is NULL.", __FILE__, __func__, __LINE__); } } @@ -494,16 +479,15 @@ DPCTLSyclEventRef DPCTLQueue_Memcpy(__dpctl_keep const DPCTLSyclQueueRef QRef, sycl::event ev; try { ev = Q->memcpy(Dest, Src, Count); - } catch (const sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "QRef passed to memcpy was NULL" << '\n'; + error_handler("QRef passed to memcpy was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -518,23 +502,21 @@ DPCTLSyclEventRef DPCTLQueue_Prefetch(__dpctl_keep DPCTLSyclQueueRef QRef, sycl::event ev; try { ev = Q->prefetch(Ptr, Count); - } catch (sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "Attempt to prefetch USM-allocation at nullptr" - << '\n'; + error_handler("Attempt to prefetch USM-allocation at nullptr.", + __FILE__, __func__, __LINE__); return nullptr; } } else { - // todo: log error - std::cerr << "QRef passed to prefetch was NULL" << '\n'; + error_handler("QRef passed to prefetch was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -548,17 +530,16 @@ DPCTLSyclEventRef DPCTLQueue_MemAdvise(__dpctl_keep DPCTLSyclQueueRef QRef, if (Q) { sycl::event ev; try { - ev = Q->mem_advise(Ptr, Count, static_cast(Advice)); - } catch (const sycl::runtime_error &re) { - // todo: log error - std::cerr << re.what() << '\n'; + ev = Q->mem_advise(Ptr, Count, Advice); + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(ev)); } else { - // todo: log error - std::cerr << "QRef passed to prefetch was NULL" << '\n'; + error_handler("QRef passed to prefetch was NULL.", __FILE__, __func__, + __LINE__); return nullptr; } } @@ -591,8 +572,7 @@ size_t DPCTLQueue_Hash(__dpctl_keep const DPCTLSyclQueueRef QRef) return hash_fn(*Q); } else { - // todo: log error - std::cerr << "Argument QRef is null" << '\n'; + error_handler("Argument QRef is NULL.", __FILE__, __func__, __LINE__); return 0; } } @@ -612,22 +592,17 @@ __dpctl_give DPCTLSyclEventRef DPCTLQueue_SubmitBarrierForEvents( for (auto i = 0ul; i < NDepEvents; ++i) cgh.depends_on(*unwrap(DepEvents[i])); - cgh.barrier(); + cgh.ext_oneapi_barrier(); }); - } catch (runtime_error &re) { - // \todo fix error handling - std::cerr << re.what() << '\n'; - return nullptr; - } catch (std::runtime_error &sre) { - std::cerr << sre.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } return wrap(new event(e)); } else { - // todo: log error - std::cerr << "Argument QRef is null" << '\n'; + error_handler("Argument QRef is NULL", __FILE__, __func__, __LINE__); return nullptr; } } diff --git a/dpctl-capi/source/dpctl_sycl_queue_manager.cpp b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp similarity index 81% rename from dpctl-capi/source/dpctl_sycl_queue_manager.cpp rename to libsyclinterface/source/dpctl_sycl_queue_manager.cpp index c43f364cf9..15379e07b3 100644 --- a/dpctl-capi/source/dpctl_sycl_queue_manager.cpp +++ b/libsyclinterface/source/dpctl_sycl_queue_manager.cpp @@ -25,6 +25,7 @@ //===----------------------------------------------------------------------===// #include "dpctl_sycl_queue_manager.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_device_manager.h" #include /* SYCL headers */ #include @@ -56,16 +57,15 @@ struct QueueManager qs.emplace_back(*unwrap(CRef), *unwrap(DRef)); } else { - std::cerr << "Fatal Error: No cached context for default " - "device.\n"; + error_handler("Fatal Error: No cached context for default " + "device.", + __FILE__, __func__, __LINE__); std::terminate(); } delete unwrap(DRef); delete unwrap(CRef); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (runtime_error const &re) { - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); } return qs; @@ -85,8 +85,8 @@ bool DPCTLQueueMgr_GlobalQueueIsCurrent() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "Error: No global queue found.\n"; + error_handler("Error: No global queue found.", __FILE__, __func__, + __LINE__); return false; } // The first entry of the QueueStack is always the global queue. If there @@ -104,8 +104,8 @@ DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "No currently active queues.\n"; + error_handler("No currently active queues.", __FILE__, __func__, + __LINE__); return nullptr; } auto last = qs.size() - 1; @@ -115,10 +115,13 @@ DPCTLSyclQueueRef DPCTLQueueMgr_GetCurrentQueue() // Relies on sycl::queue class' operator= to check for equivalent of queues. bool DPCTLQueueMgr_IsCurrentQueue(__dpctl_keep const DPCTLSyclQueueRef QRef) { + if (!QRef) { + return false; + } auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "No currently active queues.\n"; + error_handler("No currently active queues.", __FILE__, __func__, + __LINE__); return false; } auto last = qs.size() - 1; @@ -135,8 +138,9 @@ void DPCTLQueueMgr_SetGlobalQueue(__dpctl_keep const DPCTLSyclQueueRef qRef) qs[0] = *unwrap(qRef); } else { - // TODO: This should be an error and we should not fail silently. - std::cerr << "Error: Failed to set the global queue.\n"; + error_handler("Error: Failed to set the global queue.", __FILE__, + __func__, __LINE__); + std::terminate(); } } @@ -148,8 +152,9 @@ void DPCTLQueueMgr_PushQueue(__dpctl_keep const DPCTLSyclQueueRef qRef) qs.emplace_back(*unwrap(qRef)); } else { - // TODO: This should be an error and we should not fail silently. - std::cerr << "Error: Failed to set the current queue.\n"; + error_handler("Error: Failed to set the current queue.", __FILE__, + __func__, __LINE__); + std::terminate(); } } @@ -162,7 +167,7 @@ void DPCTLQueueMgr_PopQueue() // The first entry in the QueueStack is the global queue, and should not be // removed. if (qs.size() <= 1) { - std::cerr << "No queue to pop.\n"; + error_handler("No queue to pop.", __FILE__, __func__, __LINE__); return; } qs.pop_back(); @@ -172,8 +177,8 @@ size_t DPCTLQueueMgr_GetQueueStackSize() { auto &qs = QueueManager::getQueueStack(); if (qs.empty()) { - // \todo handle error - std::cerr << "Error: No global queue found.\n"; + error_handler("Error: No global queue found.", __FILE__, __func__, + __LINE__); return -1; } // The first entry of the QueueStack is always the global queue. If there diff --git a/dpctl-capi/source/dpctl_sycl_usm_interface.cpp b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp similarity index 76% rename from dpctl-capi/source/dpctl_sycl_usm_interface.cpp rename to libsyclinterface/source/dpctl_sycl_usm_interface.cpp index be1becb1f1..e6e7b2fcf0 100644 --- a/dpctl-capi/source/dpctl_sycl_usm_interface.cpp +++ b/libsyclinterface/source/dpctl_sycl_usm_interface.cpp @@ -26,6 +26,7 @@ #include "dpctl_sycl_usm_interface.h" #include "Support/CBindingWrapping.h" +#include "dpctl_error_handlers.h" #include "dpctl_sycl_device_interface.h" #include /* SYCL headers */ @@ -45,15 +46,15 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_shared(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = malloc_shared(size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -64,15 +65,15 @@ DPCTLaligned_alloc_shared(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = aligned_alloc_shared(alignment, size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -81,7 +82,7 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_host(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } // SYCL 2020 spec: for devices without aspect::usm_host_allocations: @@ -97,7 +98,7 @@ DPCTLaligned_alloc_host(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } // SYCL 2020 spec: for devices without aspect::usm_host_allocations: @@ -111,15 +112,15 @@ __dpctl_give DPCTLSyclUSMRef DPCTLmalloc_device(size_t size, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = malloc_device(size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -130,15 +131,15 @@ DPCTLaligned_alloc_device(size_t alignment, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } try { auto Q = unwrap(QRef); auto Ptr = aligned_alloc_device(alignment, size, *Q); return wrap(Ptr); - } catch (feature_not_supported const &fns) { - std::cerr << fns.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -147,11 +148,12 @@ void DPCTLfree_with_queue(__dpctl_take DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclQueueRef QRef) { if (!QRef) { - std::cerr << "Input QRef is nullptr\n"; + error_handler("Input QRef is nullptr.", __FILE__, __func__, __LINE__); return; } if (!MRef) { - std::cerr << "Input MRef is nullptr, nothing to free\n"; + error_handler("Input MRef is nullptr, nothing to free.", __FILE__, + __func__, __LINE__); return; } auto Ptr = unwrap(MRef); @@ -163,11 +165,12 @@ void DPCTLfree_with_context(__dpctl_take DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return; } if (!MRef) { - std::cerr << "Input MRef is nullptr, nothing to free\n"; + error_handler("Input MRef is nullptr, nothing to free.", __FILE__, + __func__, __LINE__); return; } auto Ptr = unwrap(MRef); @@ -179,11 +182,11 @@ const char *DPCTLUSM_GetPointerType(__dpctl_keep const DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return "unknown"; } if (!MRef) { - std::cerr << "Input MRef is nullptr\n"; + error_handler("Input MRef is nullptr.", __FILE__, __func__, __LINE__); return "unknown"; } auto Ptr = unwrap(MRef); @@ -207,11 +210,11 @@ DPCTLUSM_GetPointerDevice(__dpctl_keep const DPCTLSyclUSMRef MRef, __dpctl_keep const DPCTLSyclContextRef CRef) { if (!CRef) { - std::cerr << "Input CRef is nullptr\n"; + error_handler("Input CRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } if (!MRef) { - std::cerr << "Input MRef is nullptr\n"; + error_handler("Input MRef is nullptr.", __FILE__, __func__, __LINE__); return nullptr; } diff --git a/dpctl-capi/source/dpctl_utils.cpp b/libsyclinterface/source/dpctl_utils.cpp similarity index 100% rename from dpctl-capi/source/dpctl_utils.cpp rename to libsyclinterface/source/dpctl_utils.cpp diff --git a/dpctl-capi/source/dpctl_vector_templ.cpp b/libsyclinterface/source/dpctl_vector_templ.cpp similarity index 89% rename from dpctl-capi/source/dpctl_vector_templ.cpp rename to libsyclinterface/source/dpctl_vector_templ.cpp index 36e1bd4e65..077b6ad45d 100644 --- a/dpctl-capi/source/dpctl_vector_templ.cpp +++ b/libsyclinterface/source/dpctl_vector_templ.cpp @@ -23,8 +23,9 @@ /// the wrapper functions for vector operations. /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_vector_macros.h" #include "Support/MemOwnershipAttrs.h" +#include "dpctl_error_handlers.h" +#include "dpctl_vector_macros.h" #include #include @@ -44,8 +45,9 @@ __dpctl_give VECTOR(EL) FN(EL, Create)() try { Vec = new std::vector(); return wrap(Vec); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Vec; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -68,8 +70,9 @@ __dpctl_give VECTOR(EL) wrap(new std::remove_pointer::type(*Ref))); } return wrap(Vec); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete Vec; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } @@ -133,8 +136,8 @@ SYCLREF(EL) FN(EL, GetAt)(__dpctl_keep VECTOR(EL) VRef, size_t index) SYCLREF(EL) ret; try { ret = Vec->at(index); - } catch (std::out_of_range const &oor) { - std::cerr << oor.what() << '\n'; + } catch (std::exception const &e) { + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } auto Ref = unwrap(ret); @@ -142,10 +145,9 @@ SYCLREF(EL) FN(EL, GetAt)(__dpctl_keep VECTOR(EL) VRef, size_t index) try { elPtr = new std::remove_pointer::type(*Ref); copy = wrap(elPtr); - } catch (std::bad_alloc const &ba) { + } catch (std::exception const &e) { delete elPtr; - // \todo log error - std::cerr << ba.what() << '\n'; + error_handler(e, __FILE__, __func__, __LINE__); return nullptr; } } diff --git a/dpctl-capi/tests/CMakeLists.txt b/libsyclinterface/tests/CMakeLists.txt similarity index 86% rename from dpctl-capi/tests/CMakeLists.txt rename to libsyclinterface/tests/CMakeLists.txt index 1000634794..36037b1d4b 100644 --- a/dpctl-capi/tests/CMakeLists.txt +++ b/libsyclinterface/tests/CMakeLists.txt @@ -4,7 +4,7 @@ find_package(Threads REQUIRED) # Emulate autotools like make check target to build tests set(CMAKE_CTEST_COMMAND ctest --progress --output-on-failure -j 4) -add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND}) +add_custom_target(check COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=warning ${CMAKE_CTEST_COMMAND}) enable_testing() include_directories( @@ -35,12 +35,11 @@ if(DPCTL_GENERATE_COVERAGE) list(REMOVE_ITEM dpctl_sources "${CMAKE_CURRENT_SOURCE_DIR}/../source/dpctl_vector_templ.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../source/dpcpp_kernels.cpp" ) # Add profiling flags set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping -DDPCTL_COVERAGE" + "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping -fno-sycl-use-footer -DDPCTL_COVERAGE" ) # Add all dpctl sources into a single executable so that we can run coverage @@ -51,6 +50,10 @@ if(DPCTL_GENERATE_COVERAGE) ${dpctl_sources} ${helper_sources} ) + target_include_directories(dpctl_c_api_tests + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../helper/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + ) target_link_libraries(dpctl_c_api_tests ${CMAKE_THREAD_LIBS_INIT} GTest::GTest @@ -58,8 +61,7 @@ if(DPCTL_GENERATE_COVERAGE) ${CMAKE_DL_LIBS} ) add_custom_target(llvm-cov - COMMAND ${CMAKE_MAKE_PROGRAM} dpctl_c_api_tests - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests + COMMAND ${CMAKE_COMMAND} -E env DPCTL_VERBOSITY=warning ${CMAKE_CURRENT_BINARY_DIR}/dpctl_c_api_tests COMMAND ${LLVMProfdata_EXE} merge -sparse default.profraw @@ -72,10 +74,10 @@ if(DPCTL_GENERATE_COVERAGE) ${dpctl_sources} ${helper_sources} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS dpctl_c_api_tests ) add_custom_target(lcov-genhtml - COMMAND ${CMAKE_MAKE_PROGRAM} llvm-cov COMMAND ${LLVMCov_EXE} export -format=lcov @@ -88,12 +90,9 @@ if(DPCTL_GENERATE_COVERAGE) --output-directory ${COVERAGE_OUTPUT_DIR}/dpctl-c-api-coverage WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS llvm-cov ) else() - add_library(dpcpp_kernels - STATIC - ${CMAKE_CURRENT_SOURCE_DIR}/dpcpp_kernels.cpp - ) file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) add_executable(dpctl_c_api_tests EXCLUDE_FROM_ALL ${sources}) target_link_libraries(dpctl_c_api_tests @@ -101,7 +100,6 @@ else() GTest::GTest DPCTLSyclInterface ${LEVEL_ZERO_LIBRARY} - dpcpp_kernels ) endif() diff --git a/dpctl-capi/tests/dpcpp_kernels.hpp b/libsyclinterface/tests/dpcpp_kernels.hpp similarity index 98% rename from dpctl-capi/tests/dpcpp_kernels.hpp rename to libsyclinterface/tests/dpcpp_kernels.hpp index 099ea1a2da..f4c0934d78 100644 --- a/dpctl-capi/tests/dpcpp_kernels.hpp +++ b/libsyclinterface/tests/dpcpp_kernels.hpp @@ -1,4 +1,8 @@ #pragma once +#ifndef __SYCL_INTERNAL_API +// make sure that sycl::program is defined and implemented +#define __SYCL_INTERNAL_API +#endif #include namespace dpcpp_kernels diff --git a/dpctl-capi/tests/multi_kernel.spv b/libsyclinterface/tests/multi_kernel.spv similarity index 100% rename from dpctl-capi/tests/multi_kernel.spv rename to libsyclinterface/tests/multi_kernel.spv diff --git a/libsyclinterface/tests/test_helper.cpp b/libsyclinterface/tests/test_helper.cpp new file mode 100644 index 0000000000..ea7d638aab --- /dev/null +++ b/libsyclinterface/tests/test_helper.cpp @@ -0,0 +1,201 @@ +//===--- test_helper.cpp - Test cases for helper functions ===// +// +// Data Parallel Control (dpctl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file has unit test cases for functions defined in +/// helper/include/dpctl_utils_helper.h. +/// +//===----------------------------------------------------------------------===// + +#include "Config/dpctl_config.h" +#include "dpctl_utils_helper.h" +#include +#include +#include + +struct TestHelperFns : public ::testing::Test +{ +}; + +TEST_F(TestHelperFns, ChkDeviceTypeToStr) +{ + std::string res; + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::cpu)); + ASSERT_TRUE(res == "cpu"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::gpu)); + ASSERT_TRUE(res == "gpu"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::host)); + ASSERT_TRUE(res == "host"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::custom)); + ASSERT_TRUE(res == "custom"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::accelerator)); + ASSERT_TRUE(res == "accelerator"); + + EXPECT_NO_FATAL_FAILURE( + res = DPCTL_DeviceTypeToStr(sycl::info::device_type::all)); + ASSERT_TRUE(res == "unknown"); +} + +TEST_F(TestHelperFns, ChkStrToDeviceType) +{ + sycl::info::device_type dev_type = sycl::info::device_type::automatic; + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("cpu")); + ASSERT_TRUE(dev_type == sycl::info::device_type::cpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("gpu")); + ASSERT_TRUE(dev_type == sycl::info::device_type::gpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("host")); + ASSERT_TRUE(dev_type == sycl::info::device_type::host); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("accelerator")); + ASSERT_TRUE(dev_type == sycl::info::device_type::accelerator); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_StrToDeviceType("custom")); + ASSERT_TRUE(dev_type == sycl::info::device_type::custom); + + EXPECT_THROW(DPCTL_StrToDeviceType("invalid"), std::runtime_error); +} + +TEST_F(TestHelperFns, ChkDPCTLBackendTypeToSyclBackend) +{ + sycl::backend res = sycl::backend::ext_oneapi_level_zero; + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_CUDA)); + ASSERT_TRUE(res == sycl::backend::cuda); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_HOST)); + ASSERT_TRUE(res == sycl::backend::host); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_OPENCL)); + ASSERT_TRUE(res == sycl::backend::opencl); + + EXPECT_NO_FATAL_FAILURE(res = DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_LEVEL_ZERO)); + ASSERT_TRUE(res == sycl::backend::ext_oneapi_level_zero); + + EXPECT_THROW(DPCTL_DPCTLBackendTypeToSyclBackend( + DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND), + std::runtime_error); +} + +TEST_F(TestHelperFns, ChkSyclBackendToDPCTLBackendType) +{ + DPCTLSyclBackendType DTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclBackendToDPCTLBackendType( + sycl::backend::ext_oneapi_level_zero)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_LEVEL_ZERO); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::opencl)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_OPENCL); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::host)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_HOST); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::cuda)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_CUDA); + + EXPECT_NO_FATAL_FAILURE( + DTy = DPCTL_SyclBackendToDPCTLBackendType(sycl::backend::all)); + ASSERT_TRUE(DTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestHelperFns, ChkDPCTLDeviceTypeToSyclDeviceType) +{ + sycl::info::device_type dev_type = sycl::info::device_type::automatic; + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_CPU)); + ASSERT_TRUE(dev_type == sycl::info::device_type::cpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_GPU)); + ASSERT_TRUE(dev_type == sycl::info::device_type::gpu); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_ACCELERATOR)); + ASSERT_TRUE(dev_type == sycl::info::device_type::accelerator); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_CUSTOM)); + ASSERT_TRUE(dev_type == sycl::info::device_type::custom); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_HOST_DEVICE)); + ASSERT_TRUE(dev_type == sycl::info::device_type::host); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_AUTOMATIC)); + ASSERT_TRUE(dev_type == sycl::info::device_type::automatic); + + EXPECT_NO_FATAL_FAILURE(dev_type = DPCTL_DPCTLDeviceTypeToSyclDeviceType( + DPCTLSyclDeviceType::DPCTL_ALL)); + ASSERT_TRUE(dev_type == sycl::info::device_type::all); +} + +TEST_F(TestHelperFns, SyclDeviceTypeToDPCTLDeviceType) +{ + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE; + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::cpu)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_CPU); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::gpu)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_GPU); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::host)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_HOST_DEVICE); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::accelerator)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_ACCELERATOR); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::automatic)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_AUTOMATIC); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::all)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_ALL); + + EXPECT_NO_FATAL_FAILURE(DTy = DPCTL_SyclDeviceTypeToDPCTLDeviceType( + sycl::info::device_type::custom)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_CUSTOM); +} diff --git a/dpctl-capi/tests/test_main.cpp b/libsyclinterface/tests/test_main.cpp similarity index 100% rename from dpctl-capi/tests/test_main.cpp rename to libsyclinterface/tests/test_main.cpp diff --git a/dpctl-capi/tests/test_service.cpp b/libsyclinterface/tests/test_service.cpp similarity index 100% rename from dpctl-capi/tests/test_service.cpp rename to libsyclinterface/tests/test_service.cpp diff --git a/dpctl-capi/tests/test_sycl_context_interface.cpp b/libsyclinterface/tests/test_sycl_context_interface.cpp similarity index 77% rename from dpctl-capi/tests/test_sycl_context_interface.cpp rename to libsyclinterface/tests/test_sycl_context_interface.cpp index fb42725f23..1d609671a0 100644 --- a/dpctl-capi/tests/test_sycl_context_interface.cpp +++ b/libsyclinterface/tests/test_sycl_context_interface.cpp @@ -211,3 +211,82 @@ INSTANTIATE_TEST_SUITE_P(DPCTLContextTests, "gpu:0", "gpu:1", "1")); + +struct TestDPCTLContextNullArgs : public ::testing::Test +{ + DPCTLSyclContextRef Null_CRef = nullptr; + DPCTLSyclDeviceRef Null_DRef = nullptr; + DPCTLDeviceVectorRef Null_DVRef = nullptr; + TestDPCTLContextNullArgs() = default; + ~TestDPCTLContextNullArgs() = default; +}; + +TEST_F(TestDPCTLContextNullArgs, ChkCreate) +{ + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(CRef = DPCTLContext_Create(Null_DRef, nullptr, 0)); + ASSERT_FALSE(bool(CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkCreateFromDevices) +{ + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + CRef = DPCTLContext_CreateFromDevices(Null_DVRef, nullptr, 0)); + ASSERT_FALSE(bool(CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkAreEq) +{ + DPCTLSyclContextRef Null_C2Ref = nullptr; + bool are_eq = true; + EXPECT_NO_FATAL_FAILURE(are_eq = DPCTLContext_AreEq(Null_CRef, Null_C2Ref)); + ASSERT_FALSE(are_eq); +} + +TEST_F(TestDPCTLContextNullArgs, ChkCopy) +{ + DPCTLSyclContextRef Copied_CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_CRef = DPCTLContext_Copy(Null_CRef)); + ASSERT_FALSE(bool(Copied_CRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkGetDevices) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DVRef = DPCTLContext_GetDevices(Null_CRef)); + ASSERT_FALSE(bool(DVRef)); +} + +TEST_F(TestDPCTLContextNullArgs, ChkDeviceCount) +{ + size_t count = -1; + EXPECT_NO_FATAL_FAILURE(count = DPCTLContext_DeviceCount(Null_CRef)); + ASSERT_TRUE(count == 0); +} + +TEST_F(TestDPCTLContextNullArgs, ChkIsHost) +{ + bool is_host = true; + EXPECT_NO_FATAL_FAILURE(is_host = DPCTLContext_IsHost(Null_CRef)); + ASSERT_FALSE(is_host); +} + +TEST_F(TestDPCTLContextNullArgs, ChkHash) +{ + size_t hash = 0; + EXPECT_NO_FATAL_FAILURE(hash = DPCTLContext_Hash(Null_CRef)); + ASSERT_TRUE(hash == 0); +} + +TEST_F(TestDPCTLContextNullArgs, ChkGetBackend) +{ + DPCTLSyclBackendType BTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + EXPECT_NO_FATAL_FAILURE(BTy = DPCTLContext_GetBackend(Null_CRef)); + ASSERT_TRUE(BTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestDPCTLContextNullArgs, ChkDelete) +{ + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(Null_CRef)); +} diff --git a/dpctl-capi/tests/test_sycl_device_aspects.cpp b/libsyclinterface/tests/test_sycl_device_aspects.cpp similarity index 96% rename from dpctl-capi/tests/test_sycl_device_aspects.cpp rename to libsyclinterface/tests/test_sycl_device_aspects.cpp index 8fdc1b34f6..60a7acad77 100644 --- a/dpctl-capi/tests/test_sycl_device_aspects.cpp +++ b/libsyclinterface/tests/test_sycl_device_aspects.cpp @@ -1,8 +1,8 @@ -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_enum_types.h" +#include "dpctl_utils_helper.h" #include #include #include @@ -99,8 +99,8 @@ auto build_params() cl::sycl::aspect::usm_shared_allocations), std::make_pair("usm_restricted_shared_allocations", cl::sycl::aspect::usm_restricted_shared_allocations), - std::make_pair("usm_system_allocator", - cl::sycl::aspect::usm_system_allocator)); + std::make_pair("usm_system_allocations", + cl::sycl::aspect::usm_system_allocations)); auto pairs = build_param_pairshas(syclAspect); - } catch (cl::sycl::runtime_error const &re) { + } catch (sycl::exception const &e) { } } diff --git a/dpctl-capi/tests/test_sycl_device_interface.cpp b/libsyclinterface/tests/test_sycl_device_interface.cpp similarity index 61% rename from dpctl-capi/tests/test_sycl_device_interface.cpp rename to libsyclinterface/tests/test_sycl_device_interface.cpp index 25ff7c3f47..a55bacecd7 100644 --- a/dpctl-capi/tests/test_sycl_device_interface.cpp +++ b/libsyclinterface/tests/test_sycl_device_interface.cpp @@ -24,11 +24,11 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_platform_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include @@ -390,3 +390,286 @@ INSTANTIATE_TEST_SUITE_P(DPCTLDeviceFns, "gpu:0", "gpu:1", "1")); + +struct TestDPCTLSyclDeviceNullArgs : public ::testing::Test +{ + DPCTLSyclDeviceRef Null_DRef = nullptr; + DPCTLSyclDeviceSelectorRef Null_DSRef = nullptr; +}; + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCopy) +{ + DPCTLSyclDeviceRef Copied_DRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_DRef = DPCTLDevice_Copy(Null_DRef)); + ASSERT_FALSE(bool(Copied_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateFromSelector) +{ + DPCTLSyclDeviceRef Created_DRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Created_DRef = + DPCTLDevice_CreateFromSelector(Null_DSRef)); + ASSERT_FALSE(bool(Created_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkDelete) +{ + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(Null_DRef)); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetDeviceType) +{ + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE; + EXPECT_NO_FATAL_FAILURE(DTy = DPCTLDevice_GetDeviceType(Null_DRef)); + ASSERT_TRUE(DTy == DPCTLSyclDeviceType::DPCTL_UNKNOWN_DEVICE); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsAccelerator) +{ + bool is_acc = true; + EXPECT_NO_FATAL_FAILURE(is_acc = DPCTLDevice_IsAccelerator(Null_DRef)); + ASSERT_FALSE(is_acc); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsCPU) +{ + bool is_cpu = true; + EXPECT_NO_FATAL_FAILURE(is_cpu = DPCTLDevice_IsCPU(Null_DRef)); + ASSERT_FALSE(is_cpu); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsGPU) +{ + bool is_gpu = true; + EXPECT_NO_FATAL_FAILURE(is_gpu = DPCTLDevice_IsGPU(Null_DRef)); + ASSERT_FALSE(is_gpu); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsHost) +{ + bool is_host = true; + EXPECT_NO_FATAL_FAILURE(is_host = DPCTLDevice_IsHost(Null_DRef)); + ASSERT_FALSE(is_host); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxComputeUnits) +{ + uint32_t mcu = -1; + EXPECT_NO_FATAL_FAILURE(mcu = DPCTLDevice_GetMaxComputeUnits(Null_DRef)); + ASSERT_TRUE(mcu == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetGlobalMemSize) +{ + uint64_t gmsz = -1; + EXPECT_NO_FATAL_FAILURE(gmsz = DPCTLDevice_GetGlobalMemSize(Null_DRef)); + ASSERT_TRUE(gmsz == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetLocalMemSize) +{ + uint64_t lmsz = -1; + EXPECT_NO_FATAL_FAILURE(lmsz = DPCTLDevice_GetLocalMemSize(Null_DRef)); + ASSERT_TRUE(lmsz == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkItemDims) +{ + uint32_t md = -1; + EXPECT_NO_FATAL_FAILURE(md = DPCTLDevice_GetMaxWorkItemDims(Null_DRef)); + ASSERT_TRUE(md == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkItemSizes) +{ + size_t *sz = nullptr; + EXPECT_NO_FATAL_FAILURE(sz = DPCTLDevice_GetMaxWorkItemSizes(Null_DRef)); + ASSERT_TRUE(sz == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxWorkGroupSize) +{ + size_t m = -1; + EXPECT_NO_FATAL_FAILURE(m = DPCTLDevice_GetMaxWorkGroupSize(Null_DRef)); + ASSERT_TRUE(m == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxNumSubGroups) +{ + uint32_t nsg = -1; + EXPECT_NO_FATAL_FAILURE(nsg = DPCTLDevice_GetMaxNumSubGroups(Null_DRef)); + ASSERT_TRUE(nsg == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetPlatform) +{ + DPCTLSyclPlatformRef PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(PRef = DPCTLDevice_GetPlatform(Null_DRef)); + ASSERT_TRUE(PRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkName) +{ + const char *name = nullptr; + EXPECT_NO_FATAL_FAILURE(name = DPCTLDevice_GetName(Null_DRef)); + ASSERT_TRUE(name == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkVendor) +{ + const char *vendor = nullptr; + EXPECT_NO_FATAL_FAILURE(vendor = DPCTLDevice_GetVendor(Null_DRef)); + ASSERT_TRUE(vendor == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkDriverVersion) +{ + const char *driver_version = nullptr; + EXPECT_NO_FATAL_FAILURE(driver_version = + DPCTLDevice_GetDriverVersion(Null_DRef)); + ASSERT_TRUE(driver_version == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkIsHostUnifiedMemory) +{ + bool is_hum = true; + EXPECT_NO_FATAL_FAILURE(is_hum = + DPCTLDevice_IsHostUnifiedMemory(Null_DRef)); + ASSERT_FALSE(is_hum); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkAreEq) +{ + bool are_eq = true; + EXPECT_NO_FATAL_FAILURE(are_eq = DPCTLDevice_AreEq(Null_DRef, Null_DRef)); + ASSERT_FALSE(are_eq); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkHasAspect) +{ + bool has_fp64 = true; + EXPECT_NO_FATAL_FAILURE(has_fp64 = DPCTLDevice_HasAspect( + Null_DRef, DPCTL_SyclAspectToDPCTLAspectType( + DPCTL_StrToAspectType("fp64")))); + ASSERT_FALSE(has_fp64); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxReadWriteImageArgs) +{ + uint32_t res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetMaxReadImageArgs(Null_DRef)); + ASSERT_TRUE(res == 0); + uint32_t wes = 0; + EXPECT_NO_FATAL_FAILURE(wes = DPCTLDevice_GetMaxWriteImageArgs(Null_DRef)); + ASSERT_TRUE(wes == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetMaxImageDims) +{ + size_t res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage2dMaxWidth(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage2dMaxHeight(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxHeight(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxWidth(Null_DRef)); + ASSERT_TRUE(res == 0); + + res = 0; + EXPECT_NO_FATAL_FAILURE(res = DPCTLDevice_GetImage3dMaxDepth(Null_DRef)); + ASSERT_TRUE(res == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetSubGroupIndependentForwardProgress) +{ + bool indep_pr = true; + EXPECT_NO_FATAL_FAILURE( + indep_pr = + DPCTLDevice_GetSubGroupIndependentForwardProgress(Null_DRef)); + ASSERT_FALSE(indep_pr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetPreferredVectorWidth) +{ + uint32_t w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthChar(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthShort(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthInt(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthLong(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthHalf(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthFloat(Null_DRef)); + ASSERT_TRUE(w == 0); + + w = -1; + EXPECT_NO_FATAL_FAILURE( + w = DPCTLDevice_GetPreferredVectorWidthDouble(Null_DRef)); + ASSERT_TRUE(w == 0); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkGetParentDevice) +{ + DPCTLSyclDeviceRef pDRef = nullptr; + EXPECT_NO_FATAL_FAILURE(pDRef = DPCTLDevice_GetParentDevice(Null_DRef)); + ASSERT_TRUE(pDRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesEqually) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesEqually(Null_DRef, 2)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesByCounts) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + size_t counts[2] = {1, 1}; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByCounts(Null_DRef, counts, 2)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkCreateSubDevicesByAffinity) +{ + DPCTLDeviceVectorRef DVRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity( + Null_DRef, DPCTLPartitionAffinityDomainType::not_applicable)); + ASSERT_TRUE(DVRef == nullptr); +} + +TEST_F(TestDPCTLSyclDeviceNullArgs, ChkHash) +{ + size_t hash = 0; + EXPECT_NO_FATAL_FAILURE(hash = DPCTLDevice_Hash(Null_DRef)); + ASSERT_TRUE(hash == 0); +} diff --git a/dpctl-capi/tests/test_sycl_device_invalid_filters.cpp b/libsyclinterface/tests/test_sycl_device_invalid_filters.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_device_invalid_filters.cpp rename to libsyclinterface/tests/test_sycl_device_invalid_filters.cpp diff --git a/dpctl-capi/tests/test_sycl_device_manager.cpp b/libsyclinterface/tests/test_sycl_device_manager.cpp similarity index 87% rename from dpctl-capi/tests/test_sycl_device_manager.cpp rename to libsyclinterface/tests/test_sycl_device_manager.cpp index a7ca5ef6e1..77b7a21b07 100644 --- a/dpctl-capi/tests/test_sycl_device_manager.cpp +++ b/libsyclinterface/tests/test_sycl_device_manager.cpp @@ -24,11 +24,11 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_manager.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include @@ -139,18 +139,30 @@ INSTANTIATE_TEST_SUITE_P( struct TestGetNumDevicesForDTy : public ::testing::TestWithParam { - size_t nDevices = 0; + int param; + sycl::info::device_type sycl_dty; + TestGetNumDevicesForDTy() { - cl::sycl::info::device_type sycl_dty = - DPCTL_DPCTLDeviceTypeToSyclDeviceType( - DPCTLSyclDeviceType(GetParam())); - - auto devices = cl::sycl::device::get_devices(sycl_dty); - EXPECT_TRUE(devices.size() == DPCTLDeviceMgr_GetNumDevices(GetParam())); + param = GetParam(); + DPCTLSyclDeviceType DTy = DPCTLSyclDeviceType(param); + sycl_dty = DPCTL_DPCTLDeviceTypeToSyclDeviceType(DTy); } }; +TEST_P(TestGetNumDevicesForDTy, ChkGetNumDevices) +{ + auto devices = sycl::device::get_devices(sycl_dty); + size_t nDevices = 0; + sycl::default_selector mRanker; + for (const sycl::device &d : devices) { + if (mRanker(d) < 0) + continue; + ++nDevices; + } + EXPECT_TRUE(nDevices == DPCTLDeviceMgr_GetNumDevices(param)); +} + INSTANTIATE_TEST_SUITE_P( GetDevices, TestGetNumDevicesForDTy, @@ -162,22 +174,33 @@ INSTANTIATE_TEST_SUITE_P( struct TestGetNumDevicesForBTy : public ::testing::TestWithParam { - size_t nDevices = 0; + int param; + sycl::backend sycl_bty; TestGetNumDevicesForBTy() { - cl::sycl::backend sycl_bty = DPCTL_DPCTLBackendTypeToSyclBackend( - DPCTLSyclBackendType(GetParam())); - - auto platforms = cl::sycl::platform::get_platforms(); - for (const auto &P : platforms) { - if (P.get_backend() == sycl_bty) { - auto devices = P.get_devices(); - EXPECT_TRUE(devices.size() == - DPCTLDeviceMgr_GetNumDevices(GetParam())); + param = GetParam(); + sycl_bty = + DPCTL_DPCTLBackendTypeToSyclBackend(DPCTLSyclBackendType(param)); + } +}; + +TEST_P(TestGetNumDevicesForBTy, ChkGetNumDevices) +{ + auto platforms = cl::sycl::platform::get_platforms(); + size_t nDevices = 0; + sycl::default_selector mRanker; + for (const auto &P : platforms) { + if ((P.get_backend() == sycl_bty) || (sycl_bty == sycl::backend::all)) { + auto devices = P.get_devices(); + for (const sycl::device &d : devices) { + if (mRanker(d) < 0) + continue; + ++nDevices; } } } -}; + EXPECT_TRUE(nDevices == DPCTLDeviceMgr_GetNumDevices(param)); +} INSTANTIATE_TEST_SUITE_P( GetDevices, diff --git a/dpctl-capi/tests/test_sycl_device_selector_interface.cpp b/libsyclinterface/tests/test_sycl_device_selector_interface.cpp similarity index 97% rename from dpctl-capi/tests/test_sycl_device_selector_interface.cpp rename to libsyclinterface/tests/test_sycl_device_selector_interface.cpp index 71924d1023..592575b53a 100644 --- a/dpctl-capi/tests/test_sycl_device_selector_interface.cpp +++ b/libsyclinterface/tests/test_sycl_device_selector_interface.cpp @@ -223,6 +223,13 @@ TEST_F(TestDeviceSelectorInterface, ChkDPCTLGPUSelectorScore) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef_GPU)); EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef_CPU)); EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + DPCTLSyclDeviceSelectorRef Null_DSRef = nullptr; + DPCTLSyclDeviceRef Null_DRef = nullptr; + int score = 1; + EXPECT_NO_FATAL_FAILURE( + score = DPCTLDeviceSelector_Score(Null_DSRef, Null_DRef)); + ASSERT_TRUE(score < 0); } INSTANTIATE_TEST_SUITE_P(FilterSelectorCreation, diff --git a/dpctl-capi/tests/test_sycl_device_subdevices.cpp b/libsyclinterface/tests/test_sycl_device_subdevices.cpp similarity index 86% rename from dpctl-capi/tests/test_sycl_device_subdevices.cpp rename to libsyclinterface/tests/test_sycl_device_subdevices.cpp index 1979826bfd..ebf76fe076 100644 --- a/dpctl-capi/tests/test_sycl_device_subdevices.cpp +++ b/libsyclinterface/tests/test_sycl_device_subdevices.cpp @@ -25,13 +25,13 @@ /// //===----------------------------------------------------------------------===// -#include "../helper/include/dpctl_utils_helper.h" #include "Support/CBindingWrapping.h" #include "dpctl_sycl_device_interface.h" #include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_enum_types.h" #include "dpctl_sycl_platform_interface.h" #include "dpctl_utils.h" +#include "dpctl_utils_helper.h" #include #include @@ -149,7 +149,7 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityNotApplicable) EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } - } catch (runtime_error const &re) { + } catch (exception const &e) { } } } @@ -174,13 +174,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityNuma) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -210,13 +205,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL4Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -246,13 +236,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL3Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -282,13 +267,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL2Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -318,13 +298,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, ChkCreateSubDevicesByAffinityL1Cache) auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { @@ -355,13 +330,8 @@ TEST_P(TestDPCTLSyclDeviceInterface, auto subDevices = D->create_sub_devices< info::partition_property::partition_by_affinity_domain>(domain); expected_size = subDevices.size(); - } catch (std::bad_alloc const &ba) { - std::cerr << ba.what() << '\n'; - } catch (feature_not_supported const &fnse) { - std::cerr << fnse.what() << '\n'; - } catch (runtime_error const &re) { - // \todo log error - std::cerr << re.what() << '\n'; + } catch (std::exception const &e) { + std::cerr << e.what() << std::endl; } if (DVRef && expected_size) { diff --git a/dpctl-capi/tests/test_sycl_event_interface.cpp b/libsyclinterface/tests/test_sycl_event_interface.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_event_interface.cpp rename to libsyclinterface/tests/test_sycl_event_interface.cpp diff --git a/dpctl-capi/tests/test_sycl_kernel_interface.cpp b/libsyclinterface/tests/test_sycl_kernel_interface.cpp similarity index 95% rename from dpctl-capi/tests/test_sycl_kernel_interface.cpp rename to libsyclinterface/tests/test_sycl_kernel_interface.cpp index 3b984a4f11..3a5f5b5f0f 100644 --- a/dpctl-capi/tests/test_sycl_kernel_interface.cpp +++ b/libsyclinterface/tests/test_sycl_kernel_interface.cpp @@ -125,6 +125,14 @@ TEST_P(TestDPCTLSyclKernelInterface, CheckGetNumArgs) DPCTLKernel_Delete(AxpyKernel); } +TEST_P(TestDPCTLSyclKernelInterface, CheckNullPtrArg) +{ + DPCTLSyclKernelRef AddKernel = nullptr; + + ASSERT_EQ(DPCTLKernel_GetNumArgs(AddKernel), -1); + ASSERT_EQ(DPCTLKernel_GetFunctionName(AddKernel), nullptr); +} + INSTANTIATE_TEST_SUITE_P(TestKernelInterfaceFunctions, TestDPCTLSyclKernelInterface, ::testing::Values("opencl:gpu:0", "opencl:cpu:0")); diff --git a/dpctl-capi/tests/test_sycl_platform_interface.cpp b/libsyclinterface/tests/test_sycl_platform_interface.cpp similarity index 77% rename from dpctl-capi/tests/test_sycl_platform_interface.cpp rename to libsyclinterface/tests/test_sycl_platform_interface.cpp index 95eaec4ee1..dc874b317e 100644 --- a/dpctl-capi/tests/test_sycl_platform_interface.cpp +++ b/libsyclinterface/tests/test_sycl_platform_interface.cpp @@ -115,6 +115,58 @@ struct TestDPCTLSyclPlatformInterface } }; +struct TestDPCTLSyclPlatformNull : public ::testing::Test +{ + DPCTLSyclPlatformRef NullPRef = nullptr; + DPCTLSyclDeviceSelectorRef NullDSRef = nullptr; + + TestDPCTLSyclPlatformNull() = default; + ~TestDPCTLSyclPlatformNull() = default; +}; + +TEST_F(TestDPCTLSyclPlatformNull, ChkCopy) +{ + DPCTLSyclPlatformRef Copied_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_PRef = DPCTLPlatform_Copy(NullPRef)); + ASSERT_TRUE(Copied_PRef == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkCreateFromSelector) +{ + DPCTLSyclPlatformRef Created_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Created_PRef = + DPCTLPlatform_CreateFromSelector(NullDSRef)); + ASSERT_TRUE(Created_PRef == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetBackend) +{ + DPCTLSyclBackendType BTy = DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND; + EXPECT_NO_FATAL_FAILURE(BTy = DPCTLPlatform_GetBackend(NullPRef)); + ASSERT_TRUE(BTy == DPCTLSyclBackendType::DPCTL_UNKNOWN_BACKEND); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetName) +{ + const char *name = nullptr; + EXPECT_NO_FATAL_FAILURE(name = DPCTLPlatform_GetName(NullPRef)); + ASSERT_TRUE(name == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetVendor) +{ + const char *vendor = nullptr; + EXPECT_NO_FATAL_FAILURE(vendor = DPCTLPlatform_GetVendor(NullPRef)); + ASSERT_TRUE(vendor == nullptr); +} + +TEST_F(TestDPCTLSyclPlatformNull, ChkGetVersion) +{ + const char *version = nullptr; + EXPECT_NO_FATAL_FAILURE(version = DPCTLPlatform_GetVersion(NullPRef)); + ASSERT_TRUE(version == nullptr); +} + struct TestDPCTLSyclDefaultPlatform : public ::testing::Test { DPCTLSyclPlatformRef PRef = nullptr; @@ -163,11 +215,26 @@ TEST_P(TestDPCTLSyclPlatformInterface, ChkCopy) EXPECT_NO_FATAL_FAILURE(DPCTLPlatform_Delete(Copied_PRef)); } +TEST_P(TestDPCTLSyclPlatformInterface, ChkCopyNullArg) +{ + DPCTLSyclPlatformRef Null_PRef = nullptr; + DPCTLSyclPlatformRef Copied_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(Copied_PRef = DPCTLPlatform_Copy(Null_PRef)); + EXPECT_FALSE(bool(Copied_PRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLPlatform_Delete(Copied_PRef)); +} + TEST_P(TestDPCTLSyclPlatformInterface, ChkPrintInfo) { EXPECT_NO_FATAL_FAILURE(DPCTLPlatformMgr_PrintInfo(PRef, 0)); } +TEST_P(TestDPCTLSyclPlatformInterface, ChkPrintInfoNullArg) +{ + DPCTLSyclPlatformRef Null_PRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DPCTLPlatformMgr_PrintInfo(Null_PRef, 0)); +} + TEST_F(TestDPCTLSyclDefaultPlatform, ChkGetName) { check_platform_name(PRef); diff --git a/dpctl-capi/tests/test_sycl_platform_invalid_filters.cpp b/libsyclinterface/tests/test_sycl_platform_invalid_filters.cpp similarity index 100% rename from dpctl-capi/tests/test_sycl_platform_invalid_filters.cpp rename to libsyclinterface/tests/test_sycl_platform_invalid_filters.cpp diff --git a/dpctl-capi/tests/test_sycl_program_interface.cpp b/libsyclinterface/tests/test_sycl_program_interface.cpp similarity index 80% rename from dpctl-capi/tests/test_sycl_program_interface.cpp rename to libsyclinterface/tests/test_sycl_program_interface.cpp index 3fbe154f28..13c4468a30 100644 --- a/dpctl-capi/tests/test_sycl_program_interface.cpp +++ b/libsyclinterface/tests/test_sycl_program_interface.cpp @@ -98,17 +98,47 @@ TEST_P(TestDPCTLSyclProgramInterface, ChkCreateFromSpirv) ASSERT_TRUE(PRef != nullptr); ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "add")); ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); + ASSERT_FALSE(DPCTLProgram_HasKernel(PRef, nullptr)); +} + +TEST_P(TestDPCTLSyclProgramInterface, ChkCreateFromSpirvNull) +{ + DPCTLSyclContextRef Null_CRef = nullptr; + const void *null_spirv = nullptr; + DPCTLSyclProgramRef PRef = nullptr; + EXPECT_NO_FATAL_FAILURE( + PRef = DPCTLProgram_CreateFromSpirv(Null_CRef, null_spirv, 0, nullptr)); + ASSERT_TRUE(PRef == nullptr); +} + +TEST_P(TestDPCTLSyclProgramInterface, ChkHasKernelNullProgram) +{ + + DPCTLSyclProgramRef NullRef = nullptr; + ASSERT_FALSE(DPCTLProgram_HasKernel(NullRef, "add")); } TEST_P(TestDPCTLSyclProgramInterface, ChkGetKernel) { auto AddKernel = DPCTLProgram_GetKernel(PRef, "add"); auto AxpyKernel = DPCTLProgram_GetKernel(PRef, "axpy"); + auto NullKernel = DPCTLProgram_GetKernel(PRef, nullptr); ASSERT_TRUE(AddKernel != nullptr); ASSERT_TRUE(AxpyKernel != nullptr); + ASSERT_TRUE(NullKernel == nullptr); DPCTLKernel_Delete(AddKernel); DPCTLKernel_Delete(AxpyKernel); + EXPECT_NO_FATAL_FAILURE(DPCTLKernel_Delete(NullKernel)); +} + +TEST_P(TestDPCTLSyclProgramInterface, ChkGetKernelNullProgram) +{ + DPCTLSyclProgramRef NullRef = nullptr; + DPCTLSyclKernelRef KRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(KRef = DPCTLProgram_GetKernel(NullRef, "add")); + EXPECT_TRUE(KRef == nullptr); } struct TestOCLProgramFromSource : public ::testing::Test @@ -163,6 +193,24 @@ TEST_F(TestOCLProgramFromSource, CheckCreateFromOCLSource) ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); } +TEST_F(TestOCLProgramFromSource, CheckCreateFromOCLSourceNull) +{ + const char *InvalidCLProgramStr = R"CLC( + kernel void invalid(global foo* a, global bar* b) { + size_t index = get_global_id(0); + b[index] = a[index]; + } + )CLC"; + DPCTLSyclProgramRef PRef = nullptr; + + if (!DRef) + GTEST_SKIP_("Skipping as no OpenCL GPU device found.\n"); + + EXPECT_NO_FATAL_FAILURE(PRef = DPCTLProgram_CreateFromOCLSource( + CRef, InvalidCLProgramStr, CompileOpts);); + ASSERT_TRUE(PRef == nullptr); +} + TEST_F(TestOCLProgramFromSource, CheckGetKernelOCLSource) { if (!DRef) diff --git a/dpctl-capi/tests/test_sycl_queue_interface.cpp b/libsyclinterface/tests/test_sycl_queue_interface.cpp similarity index 99% rename from dpctl-capi/tests/test_sycl_queue_interface.cpp rename to libsyclinterface/tests/test_sycl_queue_interface.cpp index a75262eb20..fa8c3cd4f1 100644 --- a/dpctl-capi/tests/test_sycl_queue_interface.cpp +++ b/libsyclinterface/tests/test_sycl_queue_interface.cpp @@ -358,7 +358,7 @@ TEST_P(TestDPCTLQueueMemberFunctions, CheckGetBackend) EXPECT_TRUE(Backend == backend::host); break; case DPCTL_LEVEL_ZERO: - EXPECT_TRUE(Backend == backend::level_zero); + EXPECT_TRUE(Backend == backend::ext_oneapi_level_zero); break; case DPCTL_OPENCL: EXPECT_TRUE(Backend == backend::opencl); diff --git a/dpctl-capi/tests/test_sycl_queue_manager.cpp b/libsyclinterface/tests/test_sycl_queue_manager.cpp similarity index 92% rename from dpctl-capi/tests/test_sycl_queue_manager.cpp rename to libsyclinterface/tests/test_sycl_queue_manager.cpp index d655d5090c..69c339e0a8 100644 --- a/dpctl-capi/tests/test_sycl_queue_manager.cpp +++ b/libsyclinterface/tests/test_sycl_queue_manager.cpp @@ -314,3 +314,44 @@ TEST_F(TestDPCTLQueueMgrFeatures, GTEST_SKIP_("OpenCL CPU devices are needed, but were not found."); } } + +struct TestDPCTLQueueMgrNullArgs : public ::testing::Test +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + + TestDPCTLQueueMgrNullArgs() {} + ~TestDPCTLQueueMgrNullArgs() {} +}; + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkGlobalQueueIsCurrent) +{ + bool res = true; + EXPECT_NO_FATAL_FAILURE(res = DPCTLQueueMgr_GlobalQueueIsCurrent()); + ASSERT_TRUE(res == true || res == false); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkIsCurrentQueue) +{ + bool res = true; + EXPECT_NO_FATAL_FAILURE(res = DPCTLQueueMgr_IsCurrentQueue(Null_QRef)); + ASSERT_FALSE(res); +} + +#if 0 +TEST_F(TestDPCTLQueueMgrNullArgs, ChkSetGlobalQueue) +{ + EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); +} + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkPushGlobalQueue) +{ + EXPECT_DEATH(DPCTLQueueMgr_SetGlobalQueue(Null_QRef), "*"); +} +#endif + +TEST_F(TestDPCTLQueueMgrNullArgs, ChkGetQueueStackSize) +{ + size_t n = 0; + EXPECT_NO_FATAL_FAILURE(n = DPCTLQueueMgr_GetQueueStackSize()); + ASSERT_TRUE(n < size_t(-1)); +} diff --git a/dpctl-capi/tests/test_sycl_queue_submit.cpp b/libsyclinterface/tests/test_sycl_queue_submit.cpp similarity index 84% rename from dpctl-capi/tests/test_sycl_queue_submit.cpp rename to libsyclinterface/tests/test_sycl_queue_submit.cpp index 04441347f9..1176e4014d 100644 --- a/dpctl-capi/tests/test_sycl_queue_submit.cpp +++ b/libsyclinterface/tests/test_sycl_queue_submit.cpp @@ -43,6 +43,7 @@ namespace { constexpr size_t SIZE = 1024; +static_assert(SIZE % 8 == 0); DEFINE_SIMPLE_CONVERSION_FUNCTIONS(void, DPCTLSyclUSMRef); } /* end of anonymous namespace */ @@ -127,6 +128,71 @@ TEST_F(TestQueueSubmit, CheckSubmitRange_saxpy) DPCTLDeviceSelector_Delete(DSRef); } +TEST_F(TestQueueSubmit, CheckSubmitNDRange_saxpy) +{ + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + DPCTLDeviceMgr_PrintDeviceInfo(DRef); + ASSERT_TRUE(DRef); + auto QRef = + DPCTLQueue_CreateForDevice(DRef, nullptr, DPCTL_DEFAULT_PROPERTY); + ASSERT_TRUE(QRef); + auto CRef = DPCTLQueue_GetContext(QRef); + ASSERT_TRUE(CRef); + auto PRef = DPCTLProgram_CreateFromSpirv(CRef, spirvBuffer.data(), + spirvFileSize, nullptr); + ASSERT_TRUE(PRef != nullptr); + ASSERT_TRUE(DPCTLProgram_HasKernel(PRef, "axpy")); + auto AxpyKernel = DPCTLProgram_GetKernel(PRef, "axpy"); + + // Create the input args + auto a = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(a != nullptr); + auto b = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(b != nullptr); + auto c = DPCTLmalloc_shared(SIZE * sizeof(float), QRef); + ASSERT_TRUE(c != nullptr); + + auto a_ptr = reinterpret_cast(unwrap(a)); + auto b_ptr = reinterpret_cast(unwrap(b)); + // Initialize a,b + for (auto i = 0ul; i < SIZE; ++i) { + a_ptr[i] = i + 1.0; + b_ptr[i] = i + 2.0; + } + + // Create kernel args for axpy + float d = 10.0; + size_t gRange[] = {1, 1, SIZE}; + size_t lRange[] = {1, 1, 8}; + void *args2[4] = {unwrap(a), unwrap(b), unwrap(c), (void *)&d}; + DPCTLKernelArgType addKernelArgTypes[] = {DPCTL_VOID_PTR, DPCTL_VOID_PTR, + DPCTL_VOID_PTR, DPCTL_FLOAT}; + DPCTLSyclEventRef events[1]; + events[0] = DPCTLEvent_Create(); + + auto ERef = + DPCTLQueue_SubmitNDRange(AxpyKernel, QRef, args2, addKernelArgTypes, 4, + gRange, lRange, 3, events, 1); + ASSERT_TRUE(ERef != nullptr); + DPCTLQueue_Wait(QRef); + + // clean ups + DPCTLEvent_Delete(ERef); + DPCTLKernel_Delete(AxpyKernel); + DPCTLfree_with_queue((DPCTLSyclUSMRef)a, QRef); + DPCTLfree_with_queue((DPCTLSyclUSMRef)b, QRef); + DPCTLfree_with_queue((DPCTLSyclUSMRef)c, QRef); + DPCTLQueue_Delete(QRef); + DPCTLContext_Delete(CRef); + DPCTLProgram_Delete(PRef); + DPCTLDevice_Delete(DRef); + DPCTLDeviceSelector_Delete(DSRef); +} + #ifndef DPCTL_COVERAGE namespace { diff --git a/dpctl-capi/tests/test_sycl_usm_interface.cpp b/libsyclinterface/tests/test_sycl_usm_interface.cpp similarity index 58% rename from dpctl-capi/tests/test_sycl_usm_interface.cpp rename to libsyclinterface/tests/test_sycl_usm_interface.cpp index 414c37a565..c7c137c143 100644 --- a/dpctl-capi/tests/test_sycl_usm_interface.cpp +++ b/libsyclinterface/tests/test_sycl_usm_interface.cpp @@ -27,11 +27,13 @@ #include "Support/CBindingWrapping.h" #include "dpctl_sycl_context_interface.h" #include "dpctl_sycl_device_interface.h" +#include "dpctl_sycl_device_selector_interface.h" #include "dpctl_sycl_event_interface.h" #include "dpctl_sycl_queue_interface.h" #include "dpctl_sycl_queue_manager.h" #include "dpctl_sycl_usm_interface.h" #include +#include #include using namespace cl::sycl; @@ -163,3 +165,107 @@ TEST_F(TestDPCTLSyclUSMInterface, AlignedAllocHost) common_test_body(nbytes, Ptr, Q, "host"); DPCTLfree_with_queue(Ptr, Q); } + +struct TestDPCTLSyclUSMNullArgs : public ::testing::Test +{ +}; + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkMalloc) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + void *ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_shared(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_device(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLmalloc_host(512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkAlignedAlloc) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + void *ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(ptr = + DPCTLaligned_alloc_shared(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = + DPCTLaligned_alloc_device(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); + + EXPECT_NO_FATAL_FAILURE(ptr = DPCTLaligned_alloc_host(64, 512, Null_QRef)); + ASSERT_TRUE(ptr == nullptr); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkFree) +{ + DPCTLSyclQueueRef Null_QRef = nullptr; + DPCTLSyclUSMRef ptr = nullptr; + + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_queue(ptr, Null_QRef)); + + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLSyclQueueRef QRef = nullptr; + + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + EXPECT_NO_FATAL_FAILURE(QRef = DPCTLQueue_CreateForDevice( + DRef, nullptr, DPCTL_DEFAULT_PROPERTY)); + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_queue(ptr, QRef)); + + DPCTLSyclContextRef Null_CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_context(ptr, Null_CRef)); + + DPCTLSyclContextRef CRef = DPCTLQueue_GetContext(QRef); + EXPECT_NO_FATAL_FAILURE(DPCTLQueue_Delete(QRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLfree_with_context(ptr, CRef)); + + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(CRef)); +} + +TEST_F(TestDPCTLSyclUSMNullArgs, ChkPointerQueries) +{ + DPCTLSyclContextRef Null_CRef = nullptr; + DPCTLSyclUSMRef Null_MRef = nullptr; + const char *t = nullptr; + auto is_unknown = [=](const char *t) -> bool { + return strncmp(t, "unknown", 7) == 0; + }; + + EXPECT_NO_FATAL_FAILURE(t = DPCTLUSM_GetPointerType(Null_MRef, Null_CRef)); + ASSERT_TRUE(is_unknown(t)); + + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLSyclContextRef CRef = nullptr; + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLDefaultSelector_Create()); + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + ASSERT_TRUE(bool(DRef)); + + EXPECT_NO_FATAL_FAILURE(CRef = DPCTLContext_Create(DRef, nullptr, 0)); + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); + + t = nullptr; + EXPECT_NO_FATAL_FAILURE(t = DPCTLUSM_GetPointerType(Null_MRef, CRef)); + ASSERT_TRUE(is_unknown(t)); + + DPCTLSyclDeviceRef D2Ref = nullptr; + EXPECT_NO_FATAL_FAILURE(D2Ref = DPCTLUSM_GetPointerDevice(Null_MRef, CRef)); + ASSERT_TRUE(D2Ref == nullptr); + + EXPECT_NO_FATAL_FAILURE(DPCTLContext_Delete(CRef)); + + EXPECT_NO_FATAL_FAILURE( + D2Ref = DPCTLUSM_GetPointerDevice(Null_MRef, Null_CRef)); + ASSERT_TRUE(D2Ref == nullptr); +} diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 6b705a4341..ecbcffd2fd 100644 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -19,7 +19,7 @@ def build_backend( - l0_support=False, code_coverage=False, sycl_compiler_prefix=None + l0_support=False, code_coverage=False, glog=False, sycl_compiler_prefix=None ): import glob import os @@ -76,7 +76,7 @@ def build_backend( if os.path.exists(INSTALL_PREFIX): shutil.rmtree(INSTALL_PREFIX) - backends = os.path.join(dpctl_dir, "dpctl-capi") + backends = os.path.join(dpctl_dir, "libsyclinterface") ENABLE_LO_PROGRAM_CREATION = "ON" if l0_support else "OFF" @@ -99,6 +99,8 @@ def build_backend( "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang++"), ] + if glog: + cmake_compiler_args.append("-DDPCTL_ENABLE_GLOG=ON") if code_coverage: cmake_args = ( [ @@ -177,6 +179,8 @@ def build_backend( "-DCMAKE_CXX_COMPILER:PATH=" + os.path.join(DPCPP_ROOT, "bin", "clang++.exe"), ] + if glog: + cmake_compiler_args.append("-DDPCTL_ENABLE_GLOG=ON") cmake_args = ( [ "cmake", @@ -213,9 +217,15 @@ def build_backend( shutil.rmtree(include_dir) shutil.copytree( - os.path.join(dpctl_dir, "dpctl-capi", "include"), include_dir + os.path.join(dpctl_dir, "libsyclinterface", "include"), + os.path.join(include_dir, "syclinterface"), ) + for file in glob.glob( + os.path.join(dpctl_dir, "dpctl", "apis", "include", "*.h*") + ): + shutil.copy(file, include_dir) + if __name__ == "__main__": build_backend() diff --git a/scripts/build_for_develop.bat b/scripts/build_for_develop.bat deleted file mode 100644 index 0273a9d62a..0000000000 --- a/scripts/build_for_develop.bat +++ /dev/null @@ -1,66 +0,0 @@ -REM check if oneAPI has been activated, only try activating if not -dpcpp.exe --version >nul 2>&1 -IF %ERRORLEVEL% NEQ 0 ( - set ERRORLEVEL= - call "%ONEAPI_ROOT%\compiler\latest\env\vars.bat" - IF ERRORLEVEL 1 exit /b 1 -) -REM conda uses %ERRORLEVEL% but FPGA scripts can set it. So it should be reseted. -set ERRORLEVEL= - -rmdir /S /Q build_cmake -mkdir build_cmake - -rmdir /S /Q install -mkdir install -cd install -set "INSTALL_PREFIX=%cd%" - -cd ..\build_cmake - -set "DPCPP_ROOT=%ONEAPI_ROOT%\compiler\latest\windows" - -if defined USE_GTEST ( - set "_BUILD_CAPI_TEST=ON" -) else ( - set "_BUILD_CAPI_TEST=OFF" -) -cmake -G Ninja ^ - -DCMAKE_BUILD_TYPE=Release ^ - "-DCMAKE_CXX_FLAGS=-Wno-unused-function /EHa" ^ - "-DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX%" ^ - "-DCMAKE_PREFIX_PATH=%INSTALL_PREFIX%" ^ - "-DDPCPP_ROOT=%DPCPP_ROOT%" ^ - "-DCMAKE_C_COMPILER:PATH=%DPCPP_ROOT%\bin\icx.exe" ^ - "-DCMAKE_CXX_COMPILER:PATH=%DPCPP_ROOT%\bin\dpcpp.exe" ^ - "-DBUILD_CAPI_TESTS=%_BUILD_CAPI_TEST%" ^ - "%cd%\..\dpctl-capi" -IF %ERRORLEVEL% NEQ 0 exit /b 1 - -ninja -n -IF %ERRORLEVEL% NEQ 0 exit /b 1 -if defined USE_GTEST ( - ninja check - IF %ERRORLEVEL% NEQ 0 exit /b 1 -) -ninja install -IF %ERRORLEVEL% NEQ 0 exit /b 1 - -cd .. -xcopy install\lib\*.lib dpctl /E /Y -xcopy install\bin\*.dll dpctl /E /Y - -mkdir dpctl\include -xcopy dpctl-capi\include dpctl\include /E /Y - - -REM required by _sycl_core(dpctl) -set "DPCTL_SYCL_INTERFACE_LIBDIR=dpctl" -set "DPCTL_SYCL_INTERFACE_INCLDIR=dpctl\include" -set "CC=icx.exe" -set "CXX=dpcpp.exe" - -python setup.py clean --all -python setup.py build_ext --inplace develop -python -m unittest dpctl.tests -IF %ERRORLEVEL% NEQ 0 exit /b 1 diff --git a/scripts/build_for_develop.sh b/scripts/build_for_develop.sh deleted file mode 100755 index 530c2a2991..0000000000 --- a/scripts/build_for_develop.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set +xe - -python setup.py clean --all -python setup.py develop --coverage=True -pytest -q -ra --disable-warnings --cov dpctl --cov-report term-missing --pyargs dpctl -vv diff --git a/scripts/build_locally.py b/scripts/build_locally.py new file mode 100644 index 0000000000..acd1a4a6a0 --- /dev/null +++ b/scripts/build_locally.py @@ -0,0 +1,145 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + build_type="Release", + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + cmake_executable=None, + use_glog=False, +): + build_system = None + + if "linux" in sys.platform: + build_system = "Ninja" + elif sys.platform in ["win32", "cygwin"]: + build_system = "Ninja" + else: + assert False, sys.platform + " not supported" + + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + ] + if cmake_executable: + cmake_args += [ + "--cmake-executable=" + cmake_executable, + ] + cmake_args += [ + "--", + "-G", + build_system, + "-DCMAKE_BUILD_TYPE=" + build_type, + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + "-DDPCTL_ENABLE_GLOG:BOOL=" + ("ON" if use_glog else "OFF"), + ] + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + subprocess.check_call( + cmake_args, shell=False, cwd=setup_dir, env=os.environ + ) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl for in-place installation" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_true", + ) + driver.add_argument( + "--debug", + default="Release", + const="Debug", + action="store_const", + help="Set the compilation mode to debugging", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--cmake-executable", type=str, help="Path to cmake executable" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + driver.add_argument( + "--glog", + help="DPCTLSyclInterface uses Google logger", + dest="glog", + action="store_true", + ) + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" + args.compiler_root = None + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + build_type=args.debug, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + cmake_executable=args.cmake_executable, + use_glog=args.glog, + ) diff --git a/scripts/environment.yml b/scripts/environment.yml deleted file mode 100644 index 0a063c0a95..0000000000 --- a/scripts/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -channels: - - defaults -dependencies: - - python=3.7 - - conda-build diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py new file mode 100644 index 0000000000..611327689a --- /dev/null +++ b/scripts/gen_coverage.py @@ -0,0 +1,184 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + run_pytest=False, + bin_llvm=None, + gtest_config=None, +): + IS_LIN = False + + if "linux" in sys.platform: + IS_LIN = True + elif sys.platform in ["win32", "cygwin"]: + pass + else: + assert False, sys.platform + " not supported" + + if not IS_LIN: + raise RuntimeError( + "This scripts only supports coverage collection on Linux" + ) + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + "--", + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_GENERATE_COVERAGE=ON", + "-DDPCTL_BUILD_CAPI_TESTS=ON", + "-DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=" + setup_dir, + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + ] + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + env = None + if bin_llvm: + env = { + "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), + "LLVM_TOOLS_HOME": bin_llvm, + } + env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + if gtest_config: + cmake_args += ["-DCMAKE_PREFIX_PATH=" + gtest_config] + subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + cmake_build_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "cmake-build"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) + subprocess.check_call( + ["cmake", "--build", ".", "--target", "lcov-genhtml"], + cwd=cmake_build_dir, + ) + subprocess.check_call( + [ + "pytest", + "-q", + "-ra", + "--disable-warnings", + "--cov-config", + "pyproject.toml", + "--cov", + "dpctl", + "--cov-report", + "term-missing", + "--pyargs", + "dpctl", + "-vv", + ], + cwd=setup_dir, + shell=False, + ) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl and generate coverage" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--not-oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_false", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + driver.add_argument( + "--skip-pytest", + help="Run pytest and collect coverage", + dest="run_pytest", + action="store_false", + ) + driver.add_argument( + "--bin-llvm", help="Path to folder where llvm-cov can be found" + ) + driver.add_argument( + "--gtest-config", + help="Path to the GTestConfig.cmake file to locate a " + + "custom GTest installation.", + ) + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" + args.compiler_root = None + icx_path = subprocess.check_output(["which", "icx"]) + bin_dir = os.path.dirname(os.path.dirname(icx_path)) + args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + "bin_llvm", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + run_pytest=args.run_pytest, + bin_llvm=args.bin_llvm, + gtest_config=args.gtest_config, + ) diff --git a/scripts/gen_docs.py b/scripts/gen_docs.py new file mode 100644 index 0000000000..087bb32230 --- /dev/null +++ b/scripts/gen_docs.py @@ -0,0 +1,171 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def run( + use_oneapi=True, + c_compiler=None, + cxx_compiler=None, + level_zero=True, + compiler_root=None, + bin_llvm=None, + doxyrest_dir=None, +): + IS_LIN = False + + if "linux" in sys.platform: + IS_LIN = True + elif sys.platform in ["win32", "cygwin"]: + pass + else: + assert False, sys.platform + " not supported" + + if not IS_LIN: + raise RuntimeError( + "This scripts only supports coverage collection on Linux" + ) + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmake_args = [ + sys.executable, + "setup.py", + "develop", + "--", + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_C_COMPILER:PATH=" + c_compiler, + "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, + "-DDPCTL_ENABLE_LO_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), + "-DDPCTL_DPCPP_FROM_ONEAPI:BOOL=" + ("ON" if use_oneapi else "OFF"), + "-DDPCTL_GENERATE_DOCS=ON", + ] + + if doxyrest_dir: + cmake_args.append("-DDPCTL_ENABLE_DOXYREST=ON") + cmake_args.append("-DDoxyrest_DIR=" + doxyrest_dir) + + if compiler_root: + cmake_args += [ + "-DDPCTL_DPCPP_HOME_DIR:PATH=" + compiler_root, + ] + env = None + if bin_llvm: + env = { + "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), + } + env.update({k: v for k, v in os.environ.items() if k != "PATH"}) + # Install dpctl package + subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + # Get the path for the build directory + build_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "cmake-build"], + cwd=setup_dir, + ) + .decode("utf-8") + .strip("\n") + ) + # Generate docs + subprocess.check_call( + ["cmake", "--build", ".", "--target", "Sphinx"], cwd=build_dir + ) + generated_doc_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "index.html"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) + print("Generated documentation placed under ", generated_doc_dir) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Driver to build dpctl and generate coverage" + ) + driver = parser.add_argument_group(title="Coverage driver arguments") + driver.add_argument("--c-compiler", help="Name of C compiler", default=None) + driver.add_argument( + "--cxx-compiler", help="Name of C++ compiler", default=None + ) + driver.add_argument( + "--not-oneapi", + help="Is one-API installation", + dest="oneapi", + action="store_false", + ) + driver.add_argument( + "--compiler-root", type=str, help="Path to compiler home directory" + ) + driver.add_argument( + "--no-level-zero", + help="Enable Level Zero support", + dest="level_zero", + action="store_false", + ) + driver.add_argument( + "--bin-llvm", help="Path to folder where llvm-cov can be found" + ) + driver.add_argument( + "--doxyrest-root", + help=( + "Path to Doxyrest installation to use to generate Sphinx docs" + + "for libsyclinterface" + ), + ) + + args = parser.parse_args() + + if args.oneapi: + args.c_compiler = "icx" + args.cxx_compiler = "icpx" + args.compiler_root = None + icx_path = subprocess.check_output(["which", "icx"]) + bin_dir = os.path.dirname(os.path.dirname(icx_path)) + args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") + else: + args_to_validate = [ + "c_compiler", + "cxx_compiler", + "compiler_root", + "bin_llvm", + ] + for p in args_to_validate: + arg = getattr(args, p, None) + if not isinstance(arg, str): + opt_name = p.replace("_", "-") + raise RuntimeError( + f"Option {opt_name} must be provided is " + "using non-default DPC++ layout" + ) + if not os.path.exists(arg): + raise RuntimeError(f"Path {arg} must exist") + + run( + use_oneapi=args.oneapi, + c_compiler=args.c_compiler, + cxx_compiler=args.cxx_compiler, + level_zero=args.level_zero, + compiler_root=args.compiler_root, + bin_llvm=args.bin_llvm, + doxyrest_dir=args.doxyrest_root, + ) diff --git a/setup.py b/setup.py index 928a8e0010..9a26a8f211 100644 --- a/setup.py +++ b/setup.py @@ -15,444 +15,106 @@ # limitations under the License. import glob -import os import os.path +import pathlib import shutil import sys -import numpy as np -import setuptools.command.build_ext as orig_build_ext -import setuptools.command.develop as orig_develop -import setuptools.command.install as orig_install -from Cython.Build import cythonize -from setuptools import Extension, find_packages, setup +import skbuild +import skbuild.setuptools_wrap +import skbuild.utils +from setuptools import find_packages +from skbuild.command.build_py import build_py as _skbuild_build_py +from skbuild.command.install import install as _skbuild_install import versioneer -IS_WIN = False -IS_LIN = False - -if "linux" in sys.platform: - IS_LIN = True -elif sys.platform in ["win32", "cygwin"]: - IS_WIN = True -else: - assert False, "We currently do not build for " + sys.platform - -# global variable used to pass value of --coverage option of develop command -# to build_ext command -_coverage = False -dpctl_sycl_interface_lib = "dpctl" -dpctl_sycl_interface_include = r"dpctl/include" - # Get long description with open("README.md", "r", encoding="utf-8") as file: long_description = file.read() -def remove_empty(li): - return [el for el in li if el] - - -def get_sdl_cflags(): - cflags = [] - if IS_LIN: - cflags = [ - "-fstack-protector", - "-fPIC", - "-D_FORTIFY_SOURCE=2", - "-Wformat", - "-Wformat-security", - ] - # Add cflags from environment - cflags += remove_empty(os.getenv("CFLAGS", "").split(" ")) - - return cflags - - -def get_sdl_ldflags(): - ldflags = [] - if IS_LIN: - ldflags = ["-Wl,-z,noexecstack,-z,relro,-z,now"] - elif IS_WIN: - ldflags = [r"/NXCompat", r"/DynamicBase"] - # Add ldflags from environment - ldflags += remove_empty(os.getenv("LDFLAGS", "").split(" ")) - - return ldflags +def cleanup_destination(cmake_manifest): + """Delete library files from dpctl/ folder before + letting skbuild copy them over to avoid errors. + """ + _to_unlink = [] + for fn in cmake_manifest: + bn = os.path.basename(fn) + # delete + if "DPCTLSyclInterface" in bn: + lib_fn = os.path.join("dpctl", bn) + if os.path.exists(lib_fn): + _to_unlink.append(lib_fn) + for fn in _to_unlink: + pathlib.Path(fn).unlink() + return cmake_manifest -def get_other_cxxflags(): - if IS_LIN: - return ["-O3", "-std=c++17"] - elif IS_WIN: - # FIXME: These are specific to MSVC and we should first make sure - # what compiler we are using. - return [r"/Ox", r"/std:c++17"] +def _patched_copy_file( + src_file, dest_file, hide_listing=True, preserve_mode=True +): + """Copy ``src_file`` to ``dest_file`` ensuring parent directory exists. + By default, message like `creating directory /path/to/package` and + `copying directory /src/path/to/package -> path/to/package` are displayed + on standard output. Setting ``hide_listing`` to False avoids message from + being displayed. -def get_suppressed_warning_flags(): - if IS_LIN: - # PEP 590 renamed "tp_print" to "tp_vectorcall" and this causes a flood - # of deprecation warnings in the Cython generated module. This flag - # temporarily suppresses the warnings. The flag should not be needed - # once we move to Python 3.9 and/or Cython 0.30. - return ["-Wno-deprecated-declarations"] - elif IS_WIN: - return [] + NB: Patched here to not follows symbolic links + """ + # Create directory if needed + dest_dir = os.path.dirname(dest_file) + if dest_dir != "" and not os.path.exists(dest_dir): + if not hide_listing: + print("creating directory {}".format(dest_dir)) + skbuild.utils.mkdir_p(dest_dir) + # Copy file + if not hide_listing: + print("copying {} -> {}".format(src_file, dest_file)) + shutil.copyfile(src_file, dest_file, follow_symlinks=False) + shutil.copymode(src_file, dest_file, follow_symlinks=False) -def build_backend(l0_support, coverage, sycl_compiler_prefix): - import os.path - from importlib.util import module_from_spec, spec_from_file_location - - spec = spec_from_file_location( - "build_backend", os.path.join("scripts", "build_backend.py") - ) - builder_module = module_from_spec(spec) - spec.loader.exec_module(builder_module) - builder_module.build_backend( - l0_support=l0_support, - code_coverage=coverage, - sycl_compiler_prefix=sycl_compiler_prefix, - ) +skbuild.setuptools_wrap._copy_file = _patched_copy_file -def extensions(): - # Security flags - eca = get_sdl_cflags() - ela = get_sdl_ldflags() - libs = [] - libraries = [] - if IS_LIN: - libs += ["rt", "DPCTLSyclInterface"] - libraries = [dpctl_sycl_interface_lib] - runtime_library_dirs = ["$ORIGIN"] - elif IS_WIN: - libs += ["DPCTLSyclInterface"] - libraries = [dpctl_sycl_interface_lib] - runtime_library_dirs = [] - - extension_args = { - "depends": [ - dpctl_sycl_interface_include, - ], - "include_dirs": [np.get_include(), dpctl_sycl_interface_include], - "extra_compile_args": ( - eca + get_other_cxxflags() + get_suppressed_warning_flags() - ), - "extra_link_args": ela, - "libraries": libs, - "library_dirs": libraries, - "runtime_library_dirs": runtime_library_dirs, - "language": "c++", - "define_macros": [], - } +class BuildPyCmd(_skbuild_build_py): + def copy_file(self, src, dst, preserve_mode=True): + _patched_copy_file(src, dst, preserve_mode=preserve_mode) + return (dst, 1) - extensions = [ - Extension( - "dpctl._sycl_context", - [ - os.path.join("dpctl", "_sycl_context.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_device", - [ - os.path.join("dpctl", "_sycl_device.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_device_factory", - [ - os.path.join("dpctl", "_sycl_device_factory.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_event", - [ - os.path.join("dpctl", "_sycl_event.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_platform", - [ - os.path.join("dpctl", "_sycl_platform.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_queue", - [ - os.path.join("dpctl", "_sycl_queue.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl._sycl_queue_manager", - [ - os.path.join("dpctl", "_sycl_queue_manager.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.memory._memory", - [ - os.path.join("dpctl", "memory", "_memory.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.program._program", - [ - os.path.join("dpctl", "program", "_program.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.utils._compute_follows_data", - [ - os.path.join("dpctl", "utils", "_compute_follows_data.pyx"), - ], - **extension_args, - ), - Extension( - "dpctl.tensor._usmarray", - [ - os.path.join("dpctl", "tensor", "_usmarray.pyx"), - ], - depends=extension_args["depends"] - + [os.path.join("libtensor", "include", "usm_array.hpp")], - language="c++", - include_dirs=( - extension_args["include_dirs"] - + [os.path.join("libtensor", "include")] - ), - extra_compile_args=extension_args["extra_compile_args"], - extra_link_args=extension_args["extra_link_args"], - libraries=extension_args["libraries"], - library_dirs=extension_args["library_dirs"], - runtime_library_dirs=extension_args["runtime_library_dirs"], - define_macros=extension_args["define_macros"], - ), - ] - return extensions - - -class build_ext(orig_build_ext.build_ext): - description = "Build dpctl native extensions" - - def finalize_options(self): - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - super().finalize_options() +class InstallCmd(_skbuild_install): def run(self): - return super().run() - - -def get_build_py(orig_build_py): - class build_py(orig_build_py): - def run(self): - dpctl_src_dir = self.get_package_dir("dpctl") - dpctl_build_dir = os.path.join(self.build_lib, "dpctl") - os.makedirs(dpctl_build_dir, exist_ok=True) - if IS_LIN: - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.so*")): - # Check if the file already exists before copying. - # The check is needed when dealing with symlinks. - if not os.path.exists( - os.path.join(dpctl_build_dir, os.path.basename(fn)) - ): - shutil.copy( - src=fn, - dst=dpctl_build_dir, - follow_symlinks=False, - ) - elif IS_WIN: - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.lib")): - shutil.copy(src=fn, dst=dpctl_build_dir) - - for fn in glob.glob(os.path.join(dpctl_src_dir, "*.dll")): - shutil.copy(src=fn, dst=dpctl_build_dir) - else: - raise NotImplementedError("Unsupported platform") - return super().run() - - return build_py - - -class install(orig_install.install): - description = "Installs dpctl into Python prefix" - user_options = orig_install.install.user_options + [ - ( - "level-zero-support=", - None, - "Whether to enable support for program creation " - "for Level-zero backend", - ), - ( - "sycl-compiler-prefix=", - None, - "Path to SYCL compiler installation. None means " - "read it off ONEAPI_ROOT environment variable or fail.", - ), - ] - - def initialize_options(self): - super().initialize_options() - self.level_zero_support = "True" - self.sycl_compiler_prefix = None - - def finalize_options(self): - if isinstance(self.level_zero_support, str): - self.level_zero_support = self.level_zero_support.capitalize() - if self.level_zero_support in ["True", "False", "0", "1"]: - self.level_zero_support = bool(eval(self.level_zero_support)) - else: - raise ValueError( - "--level-zero-support value is invalid, use True/False" - ) - if isinstance(self.sycl_compiler_prefix, str): - if not os.path.exists(os.path.join(self.sycl_compiler_prefix)): - raise ValueError( - "--sycl-compiler-prefix expects a path " - "to an existing directory" - ) - elif self.sycl_compiler_prefix is None: - pass - else: - raise ValueError( - "--sycl-compiler-prefix value is invalid, use a " - "path to compiler installation. To use oneAPI, use the " - "default value, but remember to activate the compiler " - "environment" - ) - super().finalize_options() - - def run(self): - build_backend(self.level_zero_support, False, self.sycl_compiler_prefix) - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - cythonize(self.distribution.ext_modules) ret = super().run() - if IS_LIN: - dpctl_build_dir = os.path.join( - os.path.dirname(__file__), self.build_lib, "dpctl" - ) + if "linux" in sys.platform: + this_dir = os.path.dirname(os.path.abspath(__file__)) + dpctl_build_dir = os.path.join(this_dir, self.build_lib, "dpctl") dpctl_install_dir = os.path.join(self.install_libbase, "dpctl") for fn in glob.glob( os.path.join(dpctl_install_dir, "*DPCTLSyclInterface.so*") ): os.remove(fn) - shutil.copy( - src=os.path.join(dpctl_build_dir, os.path.basename(fn)), - dst=dpctl_install_dir, - follow_symlinks=False, - ) + base_fn = os.path.basename(fn) + src_file = os.path.join(dpctl_build_dir, base_fn) + dst_file = os.path.join(dpctl_install_dir, base_fn) + _patched_copy_file(src_file, dst_file) return ret -class develop(orig_develop.develop): - description = "Installs dpctl in place" - user_options = orig_develop.develop.user_options + [ - ( - "level-zero-support=", - None, - "Whether to enable support for program creation " - "for Level-zero backend", - ), - ( - "sycl-compiler-prefix=", - None, - "Path to SYCL compiler installation. None means " - "read it off ONEAPI_ROOT environment variable or fail.", - ), - ( - "coverage=", - None, - "Whether to generate coverage report " - "when building the backend library", - ), - ] - - def initialize_options(self): - super().initialize_options() - self.level_zero_support = "True" - self.coverage = "False" - self.sycl_compiler_prefix = None - - def finalize_options(self): - if isinstance(self.level_zero_support, str): - self.level_zero_support = self.level_zero_support.capitalize() - if self.level_zero_support in ["True", "False", "0", "1"]: - self.level_zero_support = bool(eval(self.level_zero_support)) - else: - raise ValueError( - "--level-zero-support value is invalid, use True/False" - ) - if isinstance(self.coverage, str): - self.coverage = self.coverage.capitalize() - if self.coverage in ["True", "False", "0", "1"]: - self.coverage = bool(eval(self.coverage)) - global _coverage - _coverage = self.coverage - else: - raise ValueError("--coverage value is invalid, use True/False") - if isinstance(self.sycl_compiler_prefix, str): - if not os.path.exists(os.path.join(self.sycl_compiler_prefix)): - raise ValueError( - "--sycl-compiler-prefix expects a path " - "to an existing directory" - ) - elif self.sycl_compiler_prefix is None: - pass - else: - raise ValueError( - "--sycl-compiler-prefix value is invalid, use a " - "path to compiler installation. To use oneAPI, use the " - "default value, but remember to activate the compiler " - "environment" - ) - super().finalize_options() - - def run(self): - build_backend( - self.level_zero_support, self.coverage, self.sycl_compiler_prefix - ) - if _coverage: - pre_d = getattr(self, "define", None) - if pre_d is None: - self.define = "CYTHON_TRACE" - else: - self.define = ",".join((pre_d, "CYTHON_TRACE")) - cythonize(self.distribution.ext_modules) - return super().run() - - def _get_cmdclass(): - cmdclass = versioneer.get_cmdclass() - cmdclass["build_py"] = get_build_py(cmdclass["build_py"]) - cmdclass["install"] = install - cmdclass["develop"] = develop - cmdclass["build_ext"] = build_ext + cmdclass = versioneer.get_cmdclass( + cmdclass={ + "build_py": BuildPyCmd, + "install": InstallCmd, + } + ) return cmdclass -setup( +skbuild.setup( name="dpctl", version=versioneer.get_version(), cmdclass=_get_cmdclass(), @@ -463,18 +125,30 @@ def _get_cmdclass(): author="Intel Corporation", url="https://github.com/IntelPython/dpctl", packages=find_packages(include=["*"]), + package_data={"dpctl": ["tests/*.*", "tests/helper/*.py"]}, include_package_data=True, - ext_modules=extensions(), zip_safe=False, setup_requires=["Cython"], install_requires=[ "numpy", ], + extras_require={ + "docs": [ + "Cython", + "sphinx", + "sphinx_rtd_theme", + "pydot", + "graphviz", + "sphinxcontrib-programoutput", + ], + "coverage": ["Cython", "pytest", "pytest-cov", "coverage", "tomli"], + }, keywords="dpctl", classifiers=[ "Development Status :: 3 - Alpha", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], + cmake_process_manifest_hook=cleanup_destination, ) diff --git a/versioneer.py b/versioneer.py index 64fea1c892..83dfe215b4 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.18 +# Version: 0.21 """The Versioneer - like a rocketeer, but for versions. @@ -7,16 +7,12 @@ ============== * like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer +* https://github.com/python-versioneer/python-versioneer * Brian Warner * License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) +* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] This is a tool for managing a recorded version number in distutils-based python projects. The goal is to remove the tedious and error-prone "update @@ -27,9 +23,10 @@ ## Quick Install -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) +* `pip install versioneer` to somewhere in your $PATH +* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) * run `versioneer install` in your source tree, commit the results +* Verify version information with `python setup.py version` ## Version Identifiers @@ -61,7 +58,7 @@ for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. +uncommitted changes). The version identifier is used for multiple purposes: @@ -166,7 +163,7 @@ Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). +[issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects @@ -180,7 +177,7 @@ `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. + provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs @@ -194,9 +191,9 @@ Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve @@ -224,22 +221,10 @@ cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - ## Updating Versioneer @@ -265,6 +250,14 @@ direction and include code from all supported VCS systems, reducing the number of intermediate scripts. +## Similar projects + +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer +* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools + plugin ## License @@ -274,19 +267,27 @@ Dedication" license (CC0-1.0), as described in https://creativecommons.org/publicdomain/zero/1.0/ . +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer + """ +# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring +# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements +# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error +# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with +# pylint:disable=attribute-defined-outside-init,too-many-arguments -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import errno import json import os import re import subprocess import sys +from typing import Callable, Dict class VersioneerConfig: @@ -321,12 +322,12 @@ def get_root(): # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) + my_path = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root @@ -334,30 +335,29 @@ def get_root(): def get_config_from_root(root): """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or + # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) + parser = configparser.ConfigParser() + with open(setup_cfg, "r") as cfg_file: + parser.read_file(cfg_file) VCS = parser.get("versioneer", "VCS") # mandatory - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None + # Dict-like interface for non-mandatory entries + section = parser["versioneer"] + cfg = VersioneerConfig() cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") + cfg.style = section.get("style", "") + cfg.versionfile_source = section.get("versionfile_source") + cfg.versionfile_build = section.get("versionfile_build") + cfg.tag_prefix = section.get("tag_prefix") if cfg.tag_prefix in ("''", '""'): cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") + cfg.parentdir_prefix = section.get("parentdir_prefix") + cfg.verbose = section.get("verbose") return cfg @@ -366,17 +366,15 @@ class NotThisMethod(Exception): # these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f + HANDLERS.setdefault(vcs, {})[method] = f return f return decorate @@ -385,17 +383,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -407,18 +405,16 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -426,7 +422,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# versioneer-0.21 (https://github.com/python-versioneer/python-versioneer) """Git implementation of _version.py.""" @@ -435,6 +431,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, import re import subprocess import sys +from typing import Callable, Dict def get_keywords(): @@ -472,12 +469,12 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: @@ -491,17 +488,17 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -513,15 +510,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): @@ -533,15 +528,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% @@ -558,22 +552,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -581,10 +574,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -597,11 +594,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d @@ -610,7 +607,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: @@ -619,6 +616,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %%s" %% r) return {"version": r, @@ -634,7 +636,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -642,11 +644,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"\*" - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) @@ -654,15 +658,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%%s%%s" %% (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -672,6 +677,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -688,7 +726,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces @@ -713,13 +751,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -757,19 +796,67 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%%d.dev%%d" %% (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%%d" %% (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] + rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered @@ -800,12 +887,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -876,10 +992,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -915,7 +1035,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, @@ -950,22 +1070,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -973,10 +1092,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -989,11 +1112,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1002,7 +1125,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,6 +1134,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) return {"version": r, @@ -1026,7 +1154,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -1034,11 +1162,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): version string, meaning we're inside a checked out source tree. """ GITS = ["git"] + TAG_PREFIX_REGEX = "*" if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + TAG_PREFIX_REGEX = r"*" # r"\*" - using escape on windows breaks tag extraction - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1046,15 +1176,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", + "%s%s" % (tag_prefix, TAG_PREFIX_REGEX)], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -1064,6 +1195,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -1080,7 +1244,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces @@ -1105,13 +1269,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1130,27 +1295,26 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): if ipy: files.append(ipy) try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) + my_path = __file__ + if my_path.endswith(".pyc") or my_path.endswith(".pyo"): + my_path = os.path.splitext(my_path)[0] + ".py" + versioneer_file = os.path.relpath(my_path) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) present = False try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: + with open(".gitattributes", "r") as fobj: + for line in fobj: + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + break + except OSError: pass if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() + with open(".gitattributes", "a+") as fobj: + fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) @@ -1164,15 +1328,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % @@ -1181,7 +1344,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from +# This file was generated by 'versioneer.py' (0.21) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. @@ -1203,7 +1366,7 @@ def versions_from_file(filename): try: with open(filename) as f: contents = f.read() - except EnvironmentError: + except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) @@ -1258,19 +1421,67 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version+1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered @@ -1301,12 +1512,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -1377,10 +1617,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -1480,8 +1724,12 @@ def get_version(): return get_versions()["version"] -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" +def get_cmdclass(cmdclass=None): + """Get the custom setuptools/distutils subclasses used by Versioneer. + + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and @@ -1495,9 +1743,9 @@ def get_cmdclass(): # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - cmds = {} + cmds = {} if cmdclass is None else cmdclass.copy() # we add "version" to both distutils and setuptools from distutils.core import Command @@ -1539,7 +1787,9 @@ def run(self): # setup.py egg_info -> ? # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: + if 'build_py' in cmds: + _build_py = cmds['build_py'] + elif "setuptools" in sys.modules: from setuptools.command.build_py import build_py as _build_py else: from distutils.command.build_py import build_py as _build_py @@ -1559,6 +1809,33 @@ def run(self): write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py + if 'build_ext' in cmds: + _build_ext = cmds['build_ext'] + elif "setuptools" in sys.modules: + from setuptools.command.build_ext import build_ext as _build_ext + else: + from distutils.command.build_ext import build_ext as _build_ext + + class cmd_build_ext(_build_ext): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_ext.run(self) + if self.inplace: + # build_ext --inplace will only build extensions in + # build/lib<..> dir with no _version.py to write to. + # As in place builds will already have a _version.py + # in the module dir, we do not need to write one. + return + # now locate _version.py in the new build/ directory and replace + # it with an updated value + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext + if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # nczeczulin reports that py2exe won't like the pep440-style string @@ -1592,10 +1869,7 @@ def run(self): del cmds["build_py"] if 'py2exe' in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 + from py2exe.distutils_buildexe import py2exe as _py2exe class cmd_py2exe(_py2exe): def run(self): @@ -1620,7 +1894,9 @@ def run(self): cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: + if 'sdist' in cmds: + _sdist = cmds['sdist'] + elif "setuptools" in sys.modules: from setuptools.command.sdist import sdist as _sdist else: from distutils.command.sdist import sdist as _sdist @@ -1687,21 +1963,26 @@ def make_release_tree(self, base_dir, files): """ -INIT_PY_SNIPPET = """ +OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ +INIT_PY_SNIPPET = """ +from . import {0} +__version__ = {0}.get_versions()['version'] +""" + def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" + """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: @@ -1725,12 +2006,18 @@ def do_setup(): try: with open(ipy, "r") as f: old = f.read() - except EnvironmentError: + except OSError: old = "" - if INIT_PY_SNIPPET not in old: + module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] + snippet = INIT_PY_SNIPPET.format(module) + if OLD_SNIPPET in old: + print(" replacing boilerplate in %s" % ipy) + with open(ipy, "w") as f: + f.write(old.replace(OLD_SNIPPET, snippet)) + elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) + f.write(snippet) else: print(" %s unmodified" % ipy) else: @@ -1749,7 +2036,7 @@ def do_setup(): if line.startswith("include "): for include in line.split()[1:]: simple_includes.add(include) - except EnvironmentError: + except OSError: pass # That doesn't cover everything MANIFEST.in can do # (http://docs.python.org/2/distutils/sourcedist.html#commands), so