From 14d977f2ae223a44b8c67d105a8ab3cd7f379d71 Mon Sep 17 00:00:00 2001 From: etotmeni Date: Wed, 17 Mar 2021 09:13:57 -0500 Subject: [PATCH 1/5] Added C-api for all sub-device creation funcs to device interface. --- .../helper/include/dpctl_utils_helper.h | 29 ++ .../helper/source/dpctl_utils_helper.cpp | 46 ++ .../include/dpctl_sycl_device_interface.h | 53 +++ dpctl-capi/include/dpctl_sycl_enum_types.h | 15 + .../source/dpctl_sycl_device_interface.cpp | 2 + .../tests/test_sycl_device_subdevices.cpp | 429 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 dpctl-capi/tests/test_sycl_device_subdevices.cpp diff --git a/dpctl-capi/helper/include/dpctl_utils_helper.h b/dpctl-capi/helper/include/dpctl_utils_helper.h index 8edb602060..1da40286b6 100644 --- a/dpctl-capi/helper/include/dpctl_utils_helper.h +++ b/dpctl-capi/helper/include/dpctl_utils_helper.h @@ -149,3 +149,32 @@ sycl::aspect DPCTL_DPCTLAspectTypeToSyclAspect(DPCTLSyclAspectType AspectTy); */ DPCTL_API DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(sycl::aspect Aspect); + +/*! + * @brief Converts a DPCTLPartitionAffinityDomainType enum value to its + * corresponding sycl::info::partition_affinity_domain enum value. + * + * @param PartitionAffinityDomainTy A + * DPCTLPartitionAffinityDomainType enum value + * @return A sycl::info::partition_affinity_domain enum value for the input + * DPCTLPartitionAffinityDomainType enum value. + * @throws runtime_error + */ +DPCTL_API +sycl::info::partition_affinity_domain +DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy); + +/*! + * @brief Converts a sycl::info::partition_affinity_domain enum value to + * corresponding DPCTLPartitionAffinityDomainType enum value. + * + * @param PartitionAffinityDomain sycl::info::partition_affinity_domain to be + * converted to DPCTLPartitionAffinityDomainType enum. + * @return A DPCTLPartitionAffinityDomainType enum value for the input + * sycl::info::partition_affinity_domain enum value. + * @throws runtime_error + */ +DPCTL_API +DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( + sycl::info::partition_affinity_domain PartitionAffinityDomain); diff --git a/dpctl-capi/helper/source/dpctl_utils_helper.cpp b/dpctl-capi/helper/source/dpctl_utils_helper.cpp index 20a8368881..3091393f90 100644 --- a/dpctl-capi/helper/source/dpctl_utils_helper.cpp +++ b/dpctl-capi/helper/source/dpctl_utils_helper.cpp @@ -382,3 +382,49 @@ DPCTLSyclAspectType DPCTL_SyclAspectToDPCTLAspectType(aspect Aspect) throw runtime_error("Unsupported aspect type", -1); } } + +info::partition_affinity_domain DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy) +{ + switch (PartitionAffinityDomainTy) { + case DPCTLPartitionAffinityDomainType::not_applicable: + return info::partition_affinity_domain::not_applicable; + case DPCTLPartitionAffinityDomainType::numa: + return info::partition_affinity_domain::numa; + case DPCTLPartitionAffinityDomainType::L4_cache: + return info::partition_affinity_domain::L4_cache; + case DPCTLPartitionAffinityDomainType::L3_cache: + return info::partition_affinity_domain::L3_cache; + case DPCTLPartitionAffinityDomainType::L2_cache: + return info::partition_affinity_domain::L2_cache; + case DPCTLPartitionAffinityDomainType::L1_cache: + return info::partition_affinity_domain::L1_cache; + case DPCTLPartitionAffinityDomainType::next_partitionable: + return info::partition_affinity_domain::next_partitionable; + default: + throw runtime_error("Unsupported partition_affinity_domain type", -1); + } +} + +DPCTLPartitionAffinityDomainType DPCTL_SyclPartitionAffinityDomainToDPCTLType( + sycl::info::partition_affinity_domain PartitionAffinityDomain) +{ + switch (PartitionAffinityDomain) { + case info::partition_affinity_domain::not_applicable: + return DPCTLPartitionAffinityDomainType::not_applicable; + case info::partition_affinity_domain::numa: + return DPCTLPartitionAffinityDomainType::numa; + case info::partition_affinity_domain::L4_cache: + return DPCTLPartitionAffinityDomainType::L4_cache; + case info::partition_affinity_domain::L3_cache: + return DPCTLPartitionAffinityDomainType::L3_cache; + case info::partition_affinity_domain::L2_cache: + return DPCTLPartitionAffinityDomainType::L2_cache; + case info::partition_affinity_domain::L1_cache: + return DPCTLPartitionAffinityDomainType::L1_cache; + case info::partition_affinity_domain::next_partitionable: + return DPCTLPartitionAffinityDomainType::next_partitionable; + default: + throw runtime_error("Unsupported partition_affinity_domain type", -1); + } +} diff --git a/dpctl-capi/include/dpctl_sycl_device_interface.h b/dpctl-capi/include/dpctl_sycl_device_interface.h index 1c6174031d..37d7389901 100644 --- a/dpctl-capi/include/dpctl_sycl_device_interface.h +++ b/dpctl-capi/include/dpctl_sycl_device_interface.h @@ -31,8 +31,10 @@ #include "Support/ExternC.h" #include "Support/MemOwnershipAttrs.h" #include "dpctl_data_types.h" +#include "dpctl_sycl_device_manager.h" #include "dpctl_sycl_enum_types.h" #include "dpctl_sycl_types.h" +#include DPCTL_C_EXTERN_C_BEGIN @@ -265,6 +267,57 @@ DPCTL_API bool DPCTLDevice_HasAspect(__dpctl_keep const DPCTLSyclDeviceRef DRef, DPCTLSyclAspectType AT); +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device based on the count parameter. The returned + * vector contains as many sub devices as can be created such that each sub + * device contains count compute units. If the device’s total number of compute + * units is not evenly divided by count, then the remaining compute units are + * not included in any of the sub devices. + * + * @param DRef Opaque pointer to a sycl::device + * @param count Count compute units that need to contains in + * subdevices + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, + size_t count); + +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device based on the counts parameter. For each + * non-zero value M in the counts vector, a sub device with M compute units + * is created. + * + * @param DRef Opaque pointer to a sycl::device + * @param counts Array with count compute units + * that need to contains in subdevices + * @param ncounts Number of counts + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, + __dpctl_keep size_t *counts, + size_t ncounts); + +/*! + * @brief Returns a vector of sub devices + * partitioned from this SYCL device by affinity domain based on the domain + * parameter. + * + * @param DRef Opaque pointer to a sycl::device + * @param DPCTLPartitionAffinityDomainType DPCTLPartitionAffinityDomainType + * of sycl::info::partition_affinity_domain + * @return A #DPCTLDeviceVectorRef containing #DPCTLSyclDeviceRef objects + */ +DPCTL_API +__dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( + __dpctl_keep const DPCTLSyclDeviceRef DRef, + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy); + DPCTL_C_EXTERN_C_END /*! diff --git a/dpctl-capi/include/dpctl_sycl_enum_types.h b/dpctl-capi/include/dpctl_sycl_enum_types.h index 93b9581362..9b20d3432c 100644 --- a/dpctl-capi/include/dpctl_sycl_enum_types.h +++ b/dpctl-capi/include/dpctl_sycl_enum_types.h @@ -122,6 +122,21 @@ enum DPCTLSyclAspectType usm_system_allocator }; +/*! + * @brief DPCTL analogue of sycl::info::partition_affinity_domain enum + * + */ +enum DPCTLPartitionAffinityDomainType +{ + not_applicable, + numa, + L4_cache, + L3_cache, + L2_cache, + L1_cache, + next_partitionable +}; + /*! * @brief Enums to depict the properties that can be passed to a sycl::queue * constructor. diff --git a/dpctl-capi/source/dpctl_sycl_device_interface.cpp b/dpctl-capi/source/dpctl_sycl_device_interface.cpp index 768f0a3afd..0c78f6a997 100644 --- a/dpctl-capi/source/dpctl_sycl_device_interface.cpp +++ b/dpctl-capi/source/dpctl_sycl_device_interface.cpp @@ -39,6 +39,8 @@ namespace DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device, DPCTLSyclDeviceRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device_selector, DPCTLSyclDeviceSelectorRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(platform, DPCTLSyclPlatformRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(vector_class, + DPCTLDeviceVectorRef) } /* end of anonymous namespace */ diff --git a/dpctl-capi/tests/test_sycl_device_subdevices.cpp b/dpctl-capi/tests/test_sycl_device_subdevices.cpp new file mode 100644 index 0000000000..4b7c3ea4f1 --- /dev/null +++ b/dpctl-capi/tests/test_sycl_device_subdevices.cpp @@ -0,0 +1,429 @@ +//===--- test_sycl_device_interface.cpp - Test cases for device interface ===// +// +// Data Parallel Control (dpCtl) +// +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file has unit test cases for functions defined in +/// dpctl_sycl_device_interface.h. +/// +//===----------------------------------------------------------------------===// + +#include "../helper/include/dpctl_utils_helper.h" +#include "Support/CBindingWrapping.h" +#include "dpctl_sycl_device_interface.h" +#include "dpctl_sycl_device_selector_interface.h" +#include "dpctl_sycl_enum_types.h" +#include "dpctl_sycl_platform_interface.h" +#include "dpctl_utils.h" +#include +#include + +using namespace cl::sycl; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device, DPCTLSyclDeviceRef); + +struct TestDPCTLSyclDeviceInterface + : public ::testing::TestWithParam +{ + DPCTLSyclDeviceSelectorRef DSRef = nullptr; + + TestDPCTLSyclDeviceInterface() + { + EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLFilterSelector_Create(GetParam())); + } + + void SetUp() + { + if (!DSRef) { + auto message = "Skipping as no device of type " + + std::string(GetParam()) + "."; + GTEST_SKIP_(message.c_str()); + } + } + + ~TestDPCTLSyclDeviceInterface() + { + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + } +}; + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesEqually) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + uint32_t maxCUs = 0; + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); + if (maxCUs) { + int count = maxCUs / 2; + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesEqually(DRef, count)); + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) > 0); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByCounts) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + uint32_t maxCUs = 0; + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); + if (maxCUs) { + size_t count = maxCUs / 2; + size_t *counts = nullptr; + int n = 2; + counts = new size_t[n]; + for (auto i = 0; i < n; ++i) { + counts[i] = count; + } + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByCounts(DRef, counts, n)); + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) > 0); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, + Chk_CreateSubDevicesByAffinityNotApplicable) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::not_applicable; + DPCTLPartitionAffinityDomainType dpctl_domain = + DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + auto expected_size = subDevices.size(); + + if (DVRef) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } catch (runtime_error const &re) { + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNuma) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::numa; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL4Cache) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L4_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL3Cache) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L3_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL2Cache) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L2_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL1Cache) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::L1_cache; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +TEST_P(TestDPCTLSyclDeviceInterface, + Chk_CreateSubDevicesByAffinityNextPartitionable) +{ + DPCTLSyclDeviceRef DRef = nullptr; + DPCTLDeviceVectorRef DVRef = nullptr; + + info::partition_affinity_domain domain = + info::partition_affinity_domain::next_partitionable; + DPCTLPartitionAffinityDomainType dpctl_domain; + EXPECT_NO_FATAL_FAILURE( + dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); + + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); + if (!DRef) + GTEST_SKIP_("Device not found"); + + if (dpctl_domain) { + EXPECT_NO_FATAL_FAILURE( + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); + + auto D = unwrap(DRef); + size_t expected_size = 0; + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + expected_size = subDevices.size(); + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + + if (DVRef && expected_size) { + EXPECT_TRUE(DPCTLDeviceVector_Size(DVRef) == expected_size); + EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); + } + } + + EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); +} + +INSTANTIATE_TEST_SUITE_P(DPCTLDevice_Fns, + TestDPCTLSyclDeviceInterface, + ::testing::Values("opencl", + "opencl:gpu", + "opencl:cpu", + "opencl:gpu:0", + "gpu", + "cpu", + "level_zero", + "level_zero:gpu", + "opencl:cpu:0", + "level_zero:gpu:0", + "gpu:0", + "gpu:1", + "1")); From 354522021e9fd73d6c6d99887f4daa977d5fee34 Mon Sep 17 00:00:00 2001 From: etotmeni Date: Thu, 25 Mar 2021 05:08:56 -0500 Subject: [PATCH 2/5] Add Python API changes to add sub-device creation functions. --- .../include/dpctl_sycl_device_interface.h | 1 - .../source/dpctl_sycl_device_interface.cpp | 86 ++++++++++++++ .../tests/test_sycl_device_subdevices.cpp | 75 ++---------- dpctl/_backend.pxd | 19 ++- dpctl/_sycl_device.pxd | 5 + dpctl/_sycl_device.pyx | 112 ++++++++++++++++++ dpctl/tests/test_sycl_device.py | 93 +++++++++++++++ 7 files changed, 321 insertions(+), 70 deletions(-) diff --git a/dpctl-capi/include/dpctl_sycl_device_interface.h b/dpctl-capi/include/dpctl_sycl_device_interface.h index 37d7389901..7255e37ac2 100644 --- a/dpctl-capi/include/dpctl_sycl_device_interface.h +++ b/dpctl-capi/include/dpctl_sycl_device_interface.h @@ -34,7 +34,6 @@ #include "dpctl_sycl_device_manager.h" #include "dpctl_sycl_enum_types.h" #include "dpctl_sycl_types.h" -#include DPCTL_C_EXTERN_C_BEGIN diff --git a/dpctl-capi/source/dpctl_sycl_device_interface.cpp b/dpctl-capi/source/dpctl_sycl_device_interface.cpp index 0c78f6a997..72daef4162 100644 --- a/dpctl-capi/source/dpctl_sycl_device_interface.cpp +++ b/dpctl-capi/source/dpctl_sycl_device_interface.cpp @@ -527,3 +527,89 @@ uint32_t DPCTLDevice_GetPreferredVectorWidthHalf( } return vector_width_half; } + +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesEqually(__dpctl_keep const DPCTLSyclDeviceRef DRef, + size_t count) +{ + vector_class *Devices = nullptr; + auto D = unwrap(DRef); + if (D) { + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_equally>(count); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} + +__dpctl_give DPCTLDeviceVectorRef +DPCTLDevice_CreateSubDevicesByCounts(__dpctl_keep const DPCTLSyclDeviceRef DRef, + __dpctl_keep size_t *counts, + size_t ncounts) +{ + vector_class *Devices = nullptr; + std::vector vcounts; + vcounts.assign(counts, counts + ncounts); + auto D = unwrap(DRef); + if (D) { + try { + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_counts>(vcounts); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} + +__dpctl_give DPCTLDeviceVectorRef DPCTLDevice_CreateSubDevicesByAffinity( + __dpctl_keep const DPCTLSyclDeviceRef DRef, + DPCTLPartitionAffinityDomainType PartitionAffinityDomainTy) +{ + vector_class *Devices = nullptr; + auto D = unwrap(DRef); + if (D) { + try { + auto domain = DPCTL_DPCTLPartitionAffinityDomainTypeToSycl( + PartitionAffinityDomainTy); + auto subDevices = D->create_sub_devices< + info::partition_property::partition_by_affinity_domain>(domain); + Devices = new vector_class(); + for (const auto &sd : subDevices) { + Devices->emplace_back(wrap(new device(sd))); + } + } catch (std::bad_alloc const &ba) { + std::cerr << ba.what() << '\n'; + return nullptr; + } catch (feature_not_supported const &fnse) { + std::cerr << fnse.what() << '\n'; + } catch (runtime_error const &re) { + // \todo log error + std::cerr << re.what() << '\n'; + } + } + return wrap(Devices); +} diff --git a/dpctl-capi/tests/test_sycl_device_subdevices.cpp b/dpctl-capi/tests/test_sycl_device_subdevices.cpp index 4b7c3ea4f1..bab0af5f3a 100644 --- a/dpctl-capi/tests/test_sycl_device_subdevices.cpp +++ b/dpctl-capi/tests/test_sycl_device_subdevices.cpp @@ -41,16 +41,20 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(device, DPCTLSyclDeviceRef); struct TestDPCTLSyclDeviceInterface : public ::testing::TestWithParam { - DPCTLSyclDeviceSelectorRef DSRef = nullptr; + DPCTLSyclDeviceRef DRef = nullptr; TestDPCTLSyclDeviceInterface() { - EXPECT_NO_FATAL_FAILURE(DSRef = DPCTLFilterSelector_Create(GetParam())); + auto DS = DPCTLFilterSelector_Create(GetParam()); + if (DS) { + EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DS)); + } + DPCTLDeviceSelector_Delete(DS); } void SetUp() { - if (!DSRef) { + if (!DRef) { auto message = "Skipping as no device of type " + std::string(GetParam()) + "."; GTEST_SKIP_(message.c_str()); @@ -59,20 +63,15 @@ struct TestDPCTLSyclDeviceInterface ~TestDPCTLSyclDeviceInterface() { - EXPECT_NO_FATAL_FAILURE(DPCTLDeviceSelector_Delete(DSRef)); + DPCTLDevice_Delete(DRef); } }; TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesEqually) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; uint32_t maxCUs = 0; - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); if (maxCUs) { int count = maxCUs / 2; @@ -83,20 +82,13 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesEqually) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByCounts) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; uint32_t maxCUs = 0; - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - EXPECT_NO_FATAL_FAILURE(maxCUs = DPCTLDevice_GetMaxComputeUnits(DRef)); if (maxCUs) { size_t count = maxCUs / 2; @@ -113,14 +105,11 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByCounts) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNotApplicable) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -128,10 +117,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, DPCTLPartitionAffinityDomainType dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -149,13 +134,10 @@ TEST_P(TestDPCTLSyclDeviceInterface, } catch (runtime_error const &re) { } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNuma) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -164,10 +146,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNuma) EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -192,13 +170,10 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNuma) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL4Cache) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -207,10 +182,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL4Cache) EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -235,13 +206,10 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL4Cache) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL3Cache) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -250,10 +218,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL3Cache) EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -278,13 +242,10 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL3Cache) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL2Cache) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -293,10 +254,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL2Cache) EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -321,13 +278,10 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL2Cache) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL1Cache) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -336,10 +290,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL1Cache) EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -364,14 +314,11 @@ TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityL1Cache) EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } TEST_P(TestDPCTLSyclDeviceInterface, Chk_CreateSubDevicesByAffinityNextPartitionable) { - DPCTLSyclDeviceRef DRef = nullptr; DPCTLDeviceVectorRef DVRef = nullptr; info::partition_affinity_domain domain = @@ -380,10 +327,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, EXPECT_NO_FATAL_FAILURE( dpctl_domain = DPCTL_SyclPartitionAffinityDomainToDPCTLType(domain)); - EXPECT_NO_FATAL_FAILURE(DRef = DPCTLDevice_CreateFromSelector(DSRef)); - if (!DRef) - GTEST_SKIP_("Device not found"); - if (dpctl_domain) { EXPECT_NO_FATAL_FAILURE( DVRef = DPCTLDevice_CreateSubDevicesByAffinity(DRef, dpctl_domain)); @@ -408,8 +351,6 @@ TEST_P(TestDPCTLSyclDeviceInterface, EXPECT_NO_FATAL_FAILURE(DPCTLDeviceVector_Delete(DVRef)); } } - - EXPECT_NO_FATAL_FAILURE(DPCTLDevice_Delete(DRef)); } INSTANTIATE_TEST_SUITE_P(DPCTLDevice_Fns, diff --git a/dpctl/_backend.pxd b/dpctl/_backend.pxd index d5980e3e74..5267339860 100644 --- a/dpctl/_backend.pxd +++ b/dpctl/_backend.pxd @@ -104,6 +104,18 @@ cdef extern from "dpctl_sycl_enum_types.h": ctypedef _aspect_type DPCTLSyclAspectType + cdef enum _partition_affinity_domain_type 'DPCTLPartitionAffinityDomainType': + _not_applicable 'not_applicable', + _numa 'numa', + _L4_cache 'L4_cache', + _L3_cache 'L3_cache', + _L2_cache 'L2_cache', + _L1_cache 'L1_cache', + _next_partitionable 'next_partitionable', + + ctypedef _partition_affinity_domain_type DPCTLPartitionAffinityDomainType + + cdef extern from "dpctl_sycl_types.h": cdef struct DPCTLOpaqueSyclContext cdef struct DPCTLOpaqueSyclDevice @@ -126,6 +138,11 @@ cdef extern from "dpctl_sycl_types.h": ctypedef DPCTLOpaqueSyclUSM *DPCTLSyclUSMRef +cdef extern from "dpctl_sycl_device_manager.h": + cdef struct DPCTLDeviceVector + ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef + + cdef extern from "dpctl_sycl_device_interface.h": cdef bool DPCTLDevice_AreEq(const DPCTLSyclDeviceRef DRef1, const DPCTLSyclDeviceRef DRef2) @@ -166,8 +183,6 @@ cdef extern from "dpctl_sycl_device_interface.h": cdef extern from "dpctl_sycl_device_manager.h": - cdef struct DPCTLDeviceVector - ctypedef DPCTLDeviceVector *DPCTLDeviceVectorRef cdef DPCTLDeviceVectorRef DPCTLDeviceVector_CreateFromArray( size_t nelems, DPCTLSyclDeviceRef *elems) diff --git a/dpctl/_sycl_device.pxd b/dpctl/_sycl_device.pxd index 01b8460bca..ffdcb8cf12 100644 --- a/dpctl/_sycl_device.pxd +++ b/dpctl/_sycl_device.pxd @@ -23,6 +23,7 @@ from ._backend cimport ( DPCTLSyclDeviceRef, DPCTLSyclDeviceSelectorRef, + _partition_affinity_domain_type ) @@ -44,3 +45,7 @@ cdef class SyclDevice(_SyclDevice): cdef int _init_from__SyclDevice(self, _SyclDevice other) cdef int _init_from_selector(self, DPCTLSyclDeviceSelectorRef DSRef) cdef DPCTLSyclDeviceRef get_device_ref(self) + cdef _raise_sub_devices_creation_error(self, fname, errcode) + cdef list create_sub_devices_equally(self, size_t count) + cdef list create_sub_devices_by_counts(self, list counts) + cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 9a91937102..fa7b121a6e 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -24,11 +24,16 @@ from ._backend cimport ( _aspect_type, _backend_type, _device_type, + _partition_affinity_domain_type, DPCTLCString_Delete, DPCTLDefaultSelector_Create, DPCTLDevice_Copy, DPCTLDevice_CreateFromSelector, DPCTLDevice_Delete, + DPCTLDeviceVectorRef, + DPCTLDeviceVector_Delete, + DPCTLDeviceVector_GetAt, + DPCTLDeviceVector_Size, DPCTLDevice_GetBackend, DPCTLDevice_GetDeviceType, DPCTLDevice_GetDriverInfo, @@ -61,16 +66,29 @@ from ._backend cimport ( DPCTLDevice_GetPreferredVectorWidthFloat, DPCTLDevice_GetPreferredVectorWidthDouble, DPCTLDevice_GetPreferredVectorWidthHalf, + DPCTLDevice_CreateSubDevicesEqually, + DPCTLDevice_CreateSubDevicesByCounts, + DPCTLDevice_CreateSubDevicesByAffinity, ) from . import backend_type, device_type from libc.stdint cimport uint32_t import warnings +from libc.stdlib cimport malloc, free __all__ = [ "SyclDevice", ] +cdef class SubDeviceCreationError(Exception): + """ + A SubDeviceCreationError exception is raised when + sub-devices were not created. + + """ + pass + + cdef class _SyclDevice: """ A helper metaclass to abstract a cl::sycl::device instance. """ @@ -83,6 +101,19 @@ cdef class _SyclDevice: DPCTLSize_t_Array_Delete(self._max_work_item_sizes) +cdef list _get_devices(DPCTLDeviceVectorRef DVRef): + cdef list devices = [] + cdef size_t nelems = 0 + if DVRef: + nelems = DPCTLDeviceVector_Size(DVRef) + for i in range(0, nelems): + DRef = DPCTLDeviceVector_GetAt(DVRef, i) + D = SyclDevice._create(DRef) + devices.append(D) + + return devices + + cdef class SyclDevice(_SyclDevice): """ Python equivalent for cl::sycl::device class. @@ -167,6 +198,87 @@ cdef class SyclDevice(_SyclDevice): SyclDevice._init_helper(self, DRef) return 0 + cdef _raise_sub_devices_creation_error(self, fname, errcode): + e = SubDeviceCreationError("Sub-devices were not created.") + e.fname = fname + e.code = errcode + raise e + + cdef list create_sub_devices_equally(self, size_t count): + """ Returns a vector of sub devices partitioned from this SYCL device + based on the count parameter. The returned + vector contains as many sub devices as can be created such that each sub + device contains count compute units. If the device’s total number of compute + units is not evenly divided by count, then the remaining compute units are + not included in any of the sub devices. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesEqually(self._device_ref, count) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_counts(self, list counts): + """ Returns a vector of sub devices + partitioned from this SYCL device based on the counts parameter. For each + non-zero value M in the counts vector, a sub device with M compute units + is created. + """ + cdef size_t ncounts = len(counts) + cdef size_t *counts_buff = malloc(ncounts * sizeof(size_t)) + cdef DPCTLDeviceVectorRef DVRef = NULL + cdef int i + for i in range(ncounts): + counts_buff[i] = counts[i] + DVRef = DPCTLDevice_CreateSubDevicesByCounts(self._device_ref, counts_buff, ncounts) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + free(counts_buff) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain): + """ Returns a vector of sub devices + partitioned from this SYCL device by affinity domain based on the domain + parameter. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(self._device_ref, domain) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + def create_sub_devices(self, partition): + if isinstance(partition, int) and partition > 0: + return self.create_sub_devices_equally(partition) + elif isinstance(partition, list) and all([i > 0 for i in partition]): + return self.create_sub_devices_by_counts(partition) + elif isinstance(partition, str): + if partition == "not_applicable": + domain_type = _partition_affinity_domain_type._not_applicable + elif partition == "numa": + domain_type = _partition_affinity_domain_type._numa + elif partition == "L4_cache": + domain_type = _partition_affinity_domain_type._L4_cache + elif partition == "L3_cache": + domain_type = _partition_affinity_domain_type._L3_cache + elif partition == "L2_cache": + domain_type = _partition_affinity_domain_type._L2_cache + elif partition == "L1_cache": + domain_type = _partition_affinity_domain_type._L1_cache + elif partition == "next_partitionable": + domain_type = _partition_affinity_domain_type._next_partitionable + else: + raise Exception('Unsupported type of domain') + return self.create_sub_devices_by_affinity(domain_type) + else: + raise Exception('Unsupported type of sub-device argument') + def __cinit__(self, arg=None): cdef DPCTLSyclDeviceSelectorRef DSRef = NULL cdef const char *filter_c_str = NULL diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 11ca383516..20974abd2b 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -19,6 +19,7 @@ import dpctl import pytest +from dpctl._sycl_device import SubDeviceCreationError list_of_standard_selectors = [ dpctl.select_accelerator_device, @@ -296,6 +297,89 @@ def check_get_preferred_vector_width_half(device): pytest.fail("preferred_vector_width_half call failed") +def check_create_sub_devices_equally(device): + try: + n = int(device.max_compute_units / 2) + device.create_sub_devices(partition=n) + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_counts(device): + try: + n = device.max_compute_units / 2 + device.create_sub_devices([n, n]) + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_not_applicable(device): + try: + device.create_sub_devices("not_applicable") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_numa(device): + try: + device.create_sub_devices("numa") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L4_cache(device): + try: + device.create_sub_devices("L4_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L3_cache(device): + try: + device.create_sub_devices("L3_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L2_cache(device): + try: + device.create_sub_devices("L2_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_L1_cache(device): + try: + device.create_sub_devices("L1_cache") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + +def check_create_sub_devices_by_affinity_next_partitionable(device): + try: + device.create_sub_devices("next_partitionable") + except SubDeviceCreationError: + pytest.skip("create_sub_devices can't create sub-devices on this device") + except Exception: + pytest.fail("create_sub_devices failed") + + def check_print_device_info(device): try: device.print_device_info() @@ -339,6 +423,15 @@ def check_print_device_info(device): check_has_aspect_usm_shared_allocations, check_has_aspect_usm_restricted_shared_allocations, check_has_aspect_usm_system_allocator, + check_create_sub_devices_equally, + check_create_sub_devices_by_counts, + check_create_sub_devices_by_affinity_not_applicable, + check_create_sub_devices_by_affinity_numa, + check_create_sub_devices_by_affinity_L4_cache, + check_create_sub_devices_by_affinity_L3_cache, + check_create_sub_devices_by_affinity_L2_cache, + check_create_sub_devices_by_affinity_L1_cache, + check_create_sub_devices_by_affinity_next_partitionable, check_print_device_info, ] From afafb6e6ba132ae91866a8272611b165588c80be Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 31 Mar 2021 12:44:39 -0500 Subject: [PATCH 3/5] Iterator variable must be same type as upper bound. Fixes GCC compilation warning. --- dpctl/_sycl_device.pyx | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index fa7b121a6e..4ba02641fd 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -665,3 +665,84 @@ cdef class SyclDevice(_SyclDevice): return ("".format(hex(id(self))) ) + + cdef _raise_sub_devices_creation_error(self, fname, errcode): + e = SubDeviceCreationError("Sub-devices were not created.") + e.fname = fname + e.code = errcode + raise e + + cdef list create_sub_devices_equally(self, size_t count): + """ Returns a vector of sub devices partitioned from this SYCL device + based on the count parameter. The returned + vector contains as many sub devices as can be created such that each sub + device contains count compute units. If the device’s total number of compute + units is not evenly divided by count, then the remaining compute units are + not included in any of the sub devices. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesEqually(self._device_ref, count) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_counts(self, list counts): + """ Returns a vector of sub devices + partitioned from this SYCL device based on the counts parameter. For each + non-zero value M in the counts vector, a sub device with M compute units + is created. + """ + cdef size_t ncounts = len(counts) + cdef size_t *counts_buff = malloc(ncounts * sizeof(size_t)) + cdef DPCTLDeviceVectorRef DVRef = NULL + cdef size_t i + for i in range(ncounts): + counts_buff[i] = counts[i] + DVRef = DPCTLDevice_CreateSubDevicesByCounts(self._device_ref, counts_buff, ncounts) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + free(counts_buff) + DPCTLDeviceVector_Delete(DVRef) + return devices + + cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain): + """ Returns a vector of sub devices + partitioned from this SYCL device by affinity domain based on the domain + parameter. + """ + cdef DPCTLDeviceVectorRef DVRef = NULL + DVRef = DPCTLDevice_CreateSubDevicesByAffinity(self._device_ref, domain) + if DVRef is NULL: + self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) + cdef list devices = _get_devices(DVRef) + DPCTLDeviceVector_Delete(DVRef) + return devices + + def create_sub_devices(self, partition): + if isinstance(partition, int) and partition > 0: + return self.create_sub_devices_equally(partition) + elif isinstance(partition, list) and all([i > 0 for i in partition]): + return self.create_sub_devices_by_counts(partition) + elif isinstance(partition, str): + if partition == "not_applicable": + domain_type = _partition_affinity_domain_type._not_applicable + elif partition == "numa": + domain_type = _partition_affinity_domain_type._numa + elif partition == "L4_cache": + domain_type = _partition_affinity_domain_type._L4_cache + elif partition == "L3_cache": + domain_type = _partition_affinity_domain_type._L3_cache + elif partition == "L2_cache": + domain_type = _partition_affinity_domain_type._L2_cache + elif partition == "L1_cache": + domain_type = _partition_affinity_domain_type._L1_cache + elif partition == "next_partitionable": + domain_type = _partition_affinity_domain_type._next_partitionable + else: + raise Exception('Unsupported type of domain') + return self.create_sub_devices_by_affinity(domain_type) + else: + raise Exception('Unsupported type of sub-device argument') From 179f812dfc0551a4ff8deacac8d7c2875b9b716f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 31 Mar 2021 14:01:22 -0500 Subject: [PATCH 4/5] create_sub_device changes 1. create_sub_devices requires use of keyword argument parition. Usage is d.create_sub_devices(partition=4) 2. Plugged possible memory leak in create_sub_devices_by_counts which memory allocated for array would not be freed if sub-devices vector was returned NULL 3. Changed signature of create_sub_device_by_counts from accepting object of type list, to accepting any object, which is checked to be Sized (len(obj) is expected to work), and Iterable, i.s. iteration of elements works 4. Changes to accept all integral types, such as np.int32, not just python int. 5. Got rid of _raise_create_sub_devices_exception function, and replaced it with explicit raise CreateSubDevicesError(message) --- dpctl/_sycl_device.pxd | 3 +- dpctl/_sycl_device.pyx | 69 ++++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/dpctl/_sycl_device.pxd b/dpctl/_sycl_device.pxd index ffdcb8cf12..a87b5c8196 100644 --- a/dpctl/_sycl_device.pxd +++ b/dpctl/_sycl_device.pxd @@ -45,7 +45,6 @@ cdef class SyclDevice(_SyclDevice): cdef int _init_from__SyclDevice(self, _SyclDevice other) cdef int _init_from_selector(self, DPCTLSyclDeviceSelectorRef DSRef) cdef DPCTLSyclDeviceRef get_device_ref(self) - cdef _raise_sub_devices_creation_error(self, fname, errcode) cdef list create_sub_devices_equally(self, size_t count) - cdef list create_sub_devices_by_counts(self, list counts) + cdef list create_sub_devices_by_counts(self, object counts) cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain) diff --git a/dpctl/_sycl_device.pyx b/dpctl/_sycl_device.pyx index 4ba02641fd..56cb41d8c1 100644 --- a/dpctl/_sycl_device.pyx +++ b/dpctl/_sycl_device.pyx @@ -72,8 +72,9 @@ from ._backend cimport ( ) from . import backend_type, device_type from libc.stdint cimport uint32_t -import warnings from libc.stdlib cimport malloc, free +import warnings +import collections __all__ = [ "SyclDevice", @@ -102,6 +103,9 @@ cdef class _SyclDevice: cdef list _get_devices(DPCTLDeviceVectorRef DVRef): + """ + Deletes DVRef. Pass a copy in case an original reference is needed. + """ cdef list devices = [] cdef size_t nelems = 0 if DVRef: @@ -110,6 +114,7 @@ cdef list _get_devices(DPCTLDeviceVectorRef DVRef): DRef = DPCTLDeviceVector_GetAt(DVRef, i) D = SyclDevice._create(DRef) devices.append(D) + DPCTLDeviceVector_Delete(DVRef) return devices @@ -666,12 +671,6 @@ cdef class SyclDevice(_SyclDevice): str(self.backend) + ", " + str(self.device_type) +", " + " " + self.device_name + "] at {}>".format(hex(id(self))) ) - cdef _raise_sub_devices_creation_error(self, fname, errcode): - e = SubDeviceCreationError("Sub-devices were not created.") - e.fname = fname - e.code = errcode - raise e - cdef list create_sub_devices_equally(self, size_t count): """ Returns a vector of sub devices partitioned from this SYCL device based on the count parameter. The returned @@ -683,30 +682,32 @@ cdef class SyclDevice(_SyclDevice): cdef DPCTLDeviceVectorRef DVRef = NULL DVRef = DPCTLDevice_CreateSubDevicesEqually(self._device_ref, count) if DVRef is NULL: - self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) - cdef list devices = _get_devices(DVRef) - DPCTLDeviceVector_Delete(DVRef) - return devices + raise SubDeviceCreationError("Sub-devices were not created.") + return _get_devices(DVRef) - cdef list create_sub_devices_by_counts(self, list counts): + cdef list create_sub_devices_by_counts(self, object counts): """ Returns a vector of sub devices partitioned from this SYCL device based on the counts parameter. For each non-zero value M in the counts vector, a sub device with M compute units is created. """ - cdef size_t ncounts = len(counts) - cdef size_t *counts_buff = malloc(ncounts * sizeof(size_t)) + cdef int ncounts = len(counts) + cdef size_t *counts_buff = NULL cdef DPCTLDeviceVectorRef DVRef = NULL - cdef size_t i + cdef int i + + if (ncounts == 0): + raise TypeError("Non-empty object representing list of counts is expected.") + counts_buff = malloc(( ncounts) * sizeof(size_t)) + if (counts_buff is NULL): + raise MemoryError("Allocation of counts array of size {} failed.".format(ncounts)) for i in range(ncounts): counts_buff[i] = counts[i] DVRef = DPCTLDevice_CreateSubDevicesByCounts(self._device_ref, counts_buff, ncounts) - if DVRef is NULL: - self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) - cdef list devices = _get_devices(DVRef) free(counts_buff) - DPCTLDeviceVector_Delete(DVRef) - return devices + if DVRef is NULL: + raise SubDeviceCreationError("Sub-devices were not created.") + return _get_devices(DVRef) cdef list create_sub_devices_by_affinity(self, _partition_affinity_domain_type domain): """ Returns a vector of sub devices @@ -716,16 +717,17 @@ cdef class SyclDevice(_SyclDevice): cdef DPCTLDeviceVectorRef DVRef = NULL DVRef = DPCTLDevice_CreateSubDevicesByAffinity(self._device_ref, domain) if DVRef is NULL: - self._raise_sub_devices_creation_error("DPCTLSubDeviceCreationError", -1) - cdef list devices = _get_devices(DVRef) - DPCTLDeviceVector_Delete(DVRef) - return devices - - def create_sub_devices(self, partition): + raise SubDeviceCreationError("Sub-devices were not created.") + return _get_devices(DVRef) + + def create_sub_devices(self, **kwargs): + if not kwargs.has_key('partition'): + raise TypeError("create_sub_devices(partition=parition_spec) is expected.") + partition = kwargs.pop('partition') + if (kwargs): + raise TypeError("create_sub_devices(partition=parition_spec) is expected.") if isinstance(partition, int) and partition > 0: return self.create_sub_devices_equally(partition) - elif isinstance(partition, list) and all([i > 0 for i in partition]): - return self.create_sub_devices_by_counts(partition) elif isinstance(partition, str): if partition == "not_applicable": domain_type = _partition_affinity_domain_type._not_applicable @@ -742,7 +744,14 @@ cdef class SyclDevice(_SyclDevice): elif partition == "next_partitionable": domain_type = _partition_affinity_domain_type._next_partitionable else: - raise Exception('Unsupported type of domain') + raise TypeError("Partition affinity domain {} is not understood.".format(partition)) return self.create_sub_devices_by_affinity(domain_type) + elif (isinstance(partition, collections.abc.Sized) and + isinstance(partition, collections.abc.Iterable)): + return self.create_sub_devices_by_counts(partition) else: - raise Exception('Unsupported type of sub-device argument') + try: + partition = int(partition) + return self.create_sub_devices_equally(partition) + except Exception as e: + raise TypeError("Unsupported type of sub-device argument") From 455025d99297f5786062d1a14f97866ae22f527d Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Wed, 31 Mar 2021 14:21:10 -0500 Subject: [PATCH 5/5] updated tests to use partition= keyword --- dpctl/tests/test_sycl_device.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dpctl/tests/test_sycl_device.py b/dpctl/tests/test_sycl_device.py index 20974abd2b..78b175d4bf 100644 --- a/dpctl/tests/test_sycl_device.py +++ b/dpctl/tests/test_sycl_device.py @@ -310,7 +310,7 @@ def check_create_sub_devices_equally(device): def check_create_sub_devices_by_counts(device): try: n = device.max_compute_units / 2 - device.create_sub_devices([n, n]) + device.create_sub_devices(partition=(n, n)) except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -319,7 +319,7 @@ def check_create_sub_devices_by_counts(device): def check_create_sub_devices_by_affinity_not_applicable(device): try: - device.create_sub_devices("not_applicable") + device.create_sub_devices(partition="not_applicable") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -328,7 +328,7 @@ def check_create_sub_devices_by_affinity_not_applicable(device): def check_create_sub_devices_by_affinity_numa(device): try: - device.create_sub_devices("numa") + device.create_sub_devices(partition="numa") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -337,7 +337,7 @@ def check_create_sub_devices_by_affinity_numa(device): def check_create_sub_devices_by_affinity_L4_cache(device): try: - device.create_sub_devices("L4_cache") + device.create_sub_devices(partition="L4_cache") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -346,7 +346,7 @@ def check_create_sub_devices_by_affinity_L4_cache(device): def check_create_sub_devices_by_affinity_L3_cache(device): try: - device.create_sub_devices("L3_cache") + device.create_sub_devices(partition="L3_cache") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -355,7 +355,7 @@ def check_create_sub_devices_by_affinity_L3_cache(device): def check_create_sub_devices_by_affinity_L2_cache(device): try: - device.create_sub_devices("L2_cache") + device.create_sub_devices(partition="L2_cache") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -364,7 +364,7 @@ def check_create_sub_devices_by_affinity_L2_cache(device): def check_create_sub_devices_by_affinity_L1_cache(device): try: - device.create_sub_devices("L1_cache") + device.create_sub_devices(partition="L1_cache") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: @@ -373,7 +373,7 @@ def check_create_sub_devices_by_affinity_L1_cache(device): def check_create_sub_devices_by_affinity_next_partitionable(device): try: - device.create_sub_devices("next_partitionable") + device.create_sub_devices(partition="next_partitionable") except SubDeviceCreationError: pytest.skip("create_sub_devices can't create sub-devices on this device") except Exception: