Skip to content

Enable disk caching support for OpenCL kernel binaries #2970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
9prady9 merged 3 commits into from
Jul 15, 2020
Merged

Enable disk caching support for OpenCL kernel binaries #2970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4b21f4a
Enable disk caching support for OpenCL kernel binaries
9prady9 Jul 11, 2020
c765452
Check exceptions in CUDA compileModule and Log binary write/load fail…
9prady9 Jul 11, 2020
7a049da
Use .bin extension for kernel binaries that are saved to disk
9prady9 Jul 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 62 additions & 37 deletions src/backend/cuda/compile_module.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ string getKernelCacheFilename(const int device, const string &key) {
to_string(computeFlag.first) + to_string(computeFlag.second);

return "KER" + key + "_CU_" + computeVersion + "_AF_" +
to_string(AF_API_VERSION_CURRENT) + ".cubin";
to_string(AF_API_VERSION_CURRENT) + ".bin";
}

namespace common {
Expand Down Expand Up @@ -316,40 +316,47 @@ Module compileModule(const string &moduleKey, const vector<string> &sources,
getKernelCacheFilename(device, moduleKey);
const string tempFile =
cacheDirectory + AF_PATH_SEPARATOR + makeTempFilename();

// compute CUBIN hash
const size_t cubinHash = deterministicHash(cubin, cubinSize);

// write module hash(everything: names, code & options) and CUBIN data
ofstream out(tempFile, std::ios::binary);
if (!sourceIsJIT) {
size_t mangledNamesListSize = retVal.map().size();
out.write(reinterpret_cast<const char *>(&mangledNamesListSize),
sizeof(mangledNamesListSize));
for (auto &iter : retVal.map()) {
size_t kySize = iter.first.size();
size_t vlSize = iter.second.size();
const char *key = iter.first.c_str();
const char *val = iter.second.c_str();
out.write(reinterpret_cast<const char *>(&kySize),
sizeof(kySize));
out.write(key, iter.first.size());
out.write(reinterpret_cast<const char *>(&vlSize),
sizeof(vlSize));
out.write(val, iter.second.size());
try {
// write module hash(everything: names, code & options) and CUBIN
// data
ofstream out(tempFile, std::ios::binary);
Comment thread
9prady9 marked this conversation as resolved.
if (!sourceIsJIT) {
size_t mangledNamesListSize = retVal.map().size();
out.write(reinterpret_cast<const char *>(&mangledNamesListSize),
sizeof(mangledNamesListSize));
for (auto &iter : retVal.map()) {
size_t kySize = iter.first.size();
size_t vlSize = iter.second.size();
const char *key = iter.first.c_str();
const char *val = iter.second.c_str();
out.write(reinterpret_cast<const char *>(&kySize),
sizeof(kySize));
out.write(key, iter.first.size());
out.write(reinterpret_cast<const char *>(&vlSize),
sizeof(vlSize));
out.write(val, iter.second.size());
}
}

// compute CUBIN hash
const size_t cubinHash = deterministicHash(cubin, cubinSize);

out.write(reinterpret_cast<const char *>(&cubinHash),
sizeof(cubinHash));
out.write(reinterpret_cast<const char *>(&cubinSize),
sizeof(cubinSize));
out.write(static_cast<const char *>(cubin), cubinSize);
out.close();

// try to rename temporary file into final cache file, if this fails
// this means another thread has finished compiling this kernel
// before the current thread.
if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); }
} catch (const std::ios_base::failure &e) {
AF_TRACE("{{{:<30} : failed saving binary to {} for {}, {}}}",
moduleKey, cacheFile, getDeviceProp(device).name,
e.what());
}
out.write(reinterpret_cast<const char *>(&cubinHash),
sizeof(cubinHash));
out.write(reinterpret_cast<const char *>(&cubinSize),
sizeof(cubinSize));
out.write(static_cast<const char *>(cubin), cubinSize);
out.close();

// try to rename temporary file into final cache file, if this fails
// this means another thread has finished compiling this kernel before
// the current thread.
if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); }
}
#endif

Expand Down Expand Up @@ -383,8 +390,12 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
Module retVal{nullptr};
try {
std::ifstream in(cacheFile, std::ios::binary);
if (!in.is_open()) return Module{nullptr};

if (!in.is_open()) {
AF_TRACE("{{{:<30} : Unable to open {} for {}}}", moduleKey,
cacheFile, getDeviceProp(device).name);
removeFile(cacheFile); // Remove if exists
return Module{nullptr};
}
in.exceptions(std::ios::failbit | std::ios::badbit);

if (!isJIT) {
Expand Down Expand Up @@ -430,8 +441,22 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
getDeviceProp(device).name);

retVal.set(modOut);
} catch (...) {
if (modOut != nullptr) { CU_CHECK(cuModuleUnload(modOut)); }
} catch (const std::ios_base::failure &e) {
AF_TRACE("{{{:<30} : Unable to read {} for {}}}", moduleKey, cacheFile,
getDeviceProp(device).name);
removeFile(cacheFile);
} catch (const AfError &e) {
if (e.getError() == AF_ERR_LOAD_SYM) {
AF_TRACE(
"{{{:<30} : Corrupt binary({}) found on disk for {}, removed}}",
moduleKey, cacheFile, getDeviceProp(device).name);
} else {
if (modOut != nullptr) { CU_CHECK(cuModuleUnload(modOut)); }
AF_TRACE(
"{{{:<30} : cuModuleLoadData failed with content from {} for "
"{}, {}}}",
moduleKey, cacheFile, getDeviceProp(device).name, e.what());
}
removeFile(cacheFile);
}
return retVal;
Expand Down
169 changes: 150 additions & 19 deletions src/backend/opencl/compile_module.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
#include <platform.hpp>
#include <traits.hpp>

#include <algorithm>
#include <cctype>
#include <cstdio>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
Expand All @@ -37,9 +40,12 @@ using spdlog::logger;

using std::begin;
using std::end;
using std::ofstream;
using std::ostringstream;
using std::shared_ptr;
using std::string;
using std::to_string;
using std::transform;
using std::vector;
using std::chrono::duration_cast;
using std::chrono::high_resolution_clock;
Expand All @@ -50,21 +56,30 @@ logger *getLogger() {
return logger.get();
}

#define THROW_BUILD_LOG_EXCEPTION(PROG) \
do { \
string build_error; \
build_error.reserve(4096); \
auto devices = PROG.getInfo<CL_PROGRAM_DEVICES>(); \
for (auto &device : PROG.getInfo<CL_PROGRAM_DEVICES>()) { \
build_error += \
format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n", \
device.getInfo<CL_DEVICE_NAME>(), \
PROG.getBuildInfo<CL_PROGRAM_BUILD_OPTIONS>(device), \
PROG.getBuildInfo<CL_PROGRAM_BUILD_LOG>(device)); \
} \
string info = getEnvVar("AF_OPENCL_SHOW_BUILD_INFO"); \
if (!info.empty() && info != "0") puts(build_error.c_str()); \
AF_ERROR(build_error, AF_ERR_INTERNAL); \
string getProgramBuildLog(const Program &prog) {
string build_error("");
try {
build_error.reserve(4096);
auto devices = prog.getInfo<CL_PROGRAM_DEVICES>();
for (auto &device : prog.getInfo<CL_PROGRAM_DEVICES>()) {
build_error +=
format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n",
device.getInfo<CL_DEVICE_NAME>(),
prog.getBuildInfo<CL_PROGRAM_BUILD_OPTIONS>(device),
prog.getBuildInfo<CL_PROGRAM_BUILD_LOG>(device));
}
} catch (const cl::Error &e) {
build_error = format("Failed to fetch build log: {}", e.what());
}
return build_error;
}

#define THROW_BUILD_LOG_EXCEPTION(PROG) \
do { \
string build_error = getProgramBuildLog(PROG); \
string info = getEnvVar("AF_OPENCL_SHOW_BUILD_INFO"); \
if (!info.empty() && info != "0") puts(build_error.c_str()); \
AF_ERROR(build_error, AF_ERR_INTERNAL); \
} while (0)

namespace opencl {
Expand Down Expand Up @@ -119,6 +134,21 @@ Program buildProgram(const vector<string> &kernelSources,

} // namespace opencl

string getKernelCacheFilename(const int device, const string &key) {
auto &dev = opencl::getDevice(device);

unsigned vendorId = dev.getInfo<CL_DEVICE_VENDOR_ID>();
auto devName = dev.getInfo<CL_DEVICE_NAME>();
Comment thread
9prady9 marked this conversation as resolved.
string infix = to_string(vendorId) + "_" + devName;

transform(infix.begin(), infix.end(), infix.begin(),
[](unsigned char c) { return std::toupper(c); });
std::replace(infix.begin(), infix.end(), ' ', '_');

return "KER" + key + "_CL_" + infix + "_AF_" +
to_string(AF_API_VERSION_CURRENT) + ".bin";
}

namespace common {

Module compileModule(const string &moduleKey, const vector<string> &sources,
Expand All @@ -131,6 +161,52 @@ Module compileModule(const string &moduleKey, const vector<string> &sources,
auto program = opencl::buildProgram(sources, options);
auto compileEnd = high_resolution_clock::now();

#ifdef AF_CACHE_KERNELS_TO_DISK
const int device = opencl::getActiveDeviceId();
const string &cacheDirectory = getCacheDirectory();
if (!cacheDirectory.empty()) {
const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
getKernelCacheFilename(device, moduleKey);
const string tempFile =
cacheDirectory + AF_PATH_SEPARATOR + makeTempFilename();
try {
auto binaries = program.getInfo<CL_PROGRAM_BINARIES>();

// TODO Handle cases where program objects are created from contexts
// having multiple devices
const size_t clbinSize = binaries[0].size();
const char *clbin =
reinterpret_cast<const char *>(binaries[0].data());
const size_t clbinHash = deterministicHash(clbin, clbinSize);

// write module hash and binary data to file
ofstream out(tempFile, std::ios::binary);

out.write(reinterpret_cast<const char *>(&clbinHash),
sizeof(clbinHash));
out.write(reinterpret_cast<const char *>(&clbinSize),
sizeof(clbinSize));
out.write(static_cast<const char *>(clbin), clbinSize);
out.close();
Comment thread
9prady9 marked this conversation as resolved.

// try to rename temporary file into final cache file, if this fails
// this means another thread has finished compiling this kernel
// before the current thread.
if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); }
} catch (const cl::Error &e) {
AF_TRACE("{{{:<30} : Failed to fetch opencl binary for {}, {}}}",
moduleKey,
opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
e.what());
} catch (const std::ios_base::failure &e) {
AF_TRACE("{{{:<30} : Failed writing binary to {} for {}, {}}}",
moduleKey, cacheFile,
opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
e.what());
}
}
#endif

AF_TRACE("{{{:<30} : {{ compile:{:>5} ms, {{ {} }}, {} }}}}", moduleKey,
duration_cast<milliseconds>(compileEnd - compileBegin).count(),
fmt::join(options, " "),
Expand All @@ -141,10 +217,65 @@ Module compileModule(const string &moduleKey, const vector<string> &sources,

Module loadModuleFromDisk(const int device, const string &moduleKey,
const bool isJIT) {
UNUSED(device);
UNUSED(moduleKey);
UNUSED(isJIT);
return {};
const string &cacheDirectory = getCacheDirectory();
Comment thread
9prady9 marked this conversation as resolved.
if (cacheDirectory.empty()) return Module{};

auto &dev = opencl::getDevice(device);
const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
getKernelCacheFilename(device, moduleKey);
Program program;
Module retVal{};
try {
std::ifstream in(cacheFile, std::ios::binary);
if (!in.is_open()) {
AF_ERROR("Unable to open binary cache file", AF_ERR_INTERNAL);
}
in.exceptions(std::ios::failbit | std::ios::badbit);

// TODO Handle cases where program objects are created from contexts
// having multiple devices
size_t clbinHash = 0;
in.read(reinterpret_cast<char *>(&clbinHash), sizeof(clbinHash));
size_t clbinSize = 0;
in.read(reinterpret_cast<char *>(&clbinSize), sizeof(clbinSize));
vector<unsigned char> clbin(clbinSize);
in.read(reinterpret_cast<char *>(clbin.data()), clbinSize);
in.close();
Comment thread
9prady9 marked this conversation as resolved.

const size_t recomputedHash =
deterministicHash(clbin.data(), clbinSize);
if (recomputedHash != clbinHash) {
AF_ERROR("Binary on disk seems to be corrupted", AF_ERR_LOAD_SYM);
}
program = Program(opencl::getContext(), {dev}, {clbin});
program.build();

AF_TRACE("{{{:<30} : loaded from {} for {} }}", moduleKey, cacheFile,
dev.getInfo<CL_DEVICE_NAME>());
retVal.set(program);
} catch (const AfError &e) {
if (e.getError() == AF_ERR_LOAD_SYM) {
AF_TRACE(
"{{{:<30} : Corrupt binary({}) found on disk for {}, removed}}",
moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>());
} else {
AF_TRACE("{{{:<30} : Unable to open {} for {}}}", moduleKey,
cacheFile, dev.getInfo<CL_DEVICE_NAME>());
}
removeFile(cacheFile);
} catch (const std::ios_base::failure &e) {
AF_TRACE("{{{:<30} : IO failure while loading {} for {}; {}}}",
moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>(), e.what());
removeFile(cacheFile);
} catch (const cl::Error &e) {
AF_TRACE(
"{{{:<30} : Loading OpenCL binary({}) failed for {}; {}, Build "
"Log: {}}}",
moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>(), e.what(),
getProgramBuildLog(program));
removeFile(cacheFile);
}
return retVal;
}

Kernel getKernel(const Module &mod, const string &nameExpr,
Expand Down