diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp index 1f54aa8079..38e2fed991 100644 --- a/src/backend/cuda/compile_module.cpp +++ b/src/backend/cuda/compile_module.cpp @@ -131,7 +131,7 @@ string getKernelCacheFilename(const int device, const string &key) { to_string(computeFlag.first) + to_string(computeFlag.second); return "KER" + key + "_CU_" + computeVersion + "_AF_" + - to_string(AF_API_VERSION_CURRENT) + ".cubin"; + to_string(AF_API_VERSION_CURRENT) + ".bin"; } namespace common { @@ -316,40 +316,47 @@ Module compileModule(const string &moduleKey, const vector &sources, getKernelCacheFilename(device, moduleKey); const string tempFile = cacheDirectory + AF_PATH_SEPARATOR + makeTempFilename(); - - // compute CUBIN hash - const size_t cubinHash = deterministicHash(cubin, cubinSize); - - // write module hash(everything: names, code & options) and CUBIN data - ofstream out(tempFile, std::ios::binary); - if (!sourceIsJIT) { - size_t mangledNamesListSize = retVal.map().size(); - out.write(reinterpret_cast(&mangledNamesListSize), - sizeof(mangledNamesListSize)); - for (auto &iter : retVal.map()) { - size_t kySize = iter.first.size(); - size_t vlSize = iter.second.size(); - const char *key = iter.first.c_str(); - const char *val = iter.second.c_str(); - out.write(reinterpret_cast(&kySize), - sizeof(kySize)); - out.write(key, iter.first.size()); - out.write(reinterpret_cast(&vlSize), - sizeof(vlSize)); - out.write(val, iter.second.size()); + try { + // write module hash(everything: names, code & options) and CUBIN + // data + ofstream out(tempFile, std::ios::binary); + if (!sourceIsJIT) { + size_t mangledNamesListSize = retVal.map().size(); + out.write(reinterpret_cast(&mangledNamesListSize), + sizeof(mangledNamesListSize)); + for (auto &iter : retVal.map()) { + size_t kySize = iter.first.size(); + size_t vlSize = iter.second.size(); + const char *key = iter.first.c_str(); + const char *val = iter.second.c_str(); + out.write(reinterpret_cast(&kySize), + sizeof(kySize)); + out.write(key, iter.first.size()); + out.write(reinterpret_cast(&vlSize), + sizeof(vlSize)); + out.write(val, iter.second.size()); + } } + + // compute CUBIN hash + const size_t cubinHash = deterministicHash(cubin, cubinSize); + + out.write(reinterpret_cast(&cubinHash), + sizeof(cubinHash)); + out.write(reinterpret_cast(&cubinSize), + sizeof(cubinSize)); + out.write(static_cast(cubin), cubinSize); + out.close(); + + // try to rename temporary file into final cache file, if this fails + // this means another thread has finished compiling this kernel + // before the current thread. + if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); } + } catch (const std::ios_base::failure &e) { + AF_TRACE("{{{:<30} : failed saving binary to {} for {}, {}}}", + moduleKey, cacheFile, getDeviceProp(device).name, + e.what()); } - out.write(reinterpret_cast(&cubinHash), - sizeof(cubinHash)); - out.write(reinterpret_cast(&cubinSize), - sizeof(cubinSize)); - out.write(static_cast(cubin), cubinSize); - out.close(); - - // try to rename temporary file into final cache file, if this fails - // this means another thread has finished compiling this kernel before - // the current thread. - if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); } } #endif @@ -383,8 +390,12 @@ Module loadModuleFromDisk(const int device, const string &moduleKey, Module retVal{nullptr}; try { std::ifstream in(cacheFile, std::ios::binary); - if (!in.is_open()) return Module{nullptr}; - + if (!in.is_open()) { + AF_TRACE("{{{:<30} : Unable to open {} for {}}}", moduleKey, + cacheFile, getDeviceProp(device).name); + removeFile(cacheFile); // Remove if exists + return Module{nullptr}; + } in.exceptions(std::ios::failbit | std::ios::badbit); if (!isJIT) { @@ -430,8 +441,22 @@ Module loadModuleFromDisk(const int device, const string &moduleKey, getDeviceProp(device).name); retVal.set(modOut); - } catch (...) { - if (modOut != nullptr) { CU_CHECK(cuModuleUnload(modOut)); } + } catch (const std::ios_base::failure &e) { + AF_TRACE("{{{:<30} : Unable to read {} for {}}}", moduleKey, cacheFile, + getDeviceProp(device).name); + removeFile(cacheFile); + } catch (const AfError &e) { + if (e.getError() == AF_ERR_LOAD_SYM) { + AF_TRACE( + "{{{:<30} : Corrupt binary({}) found on disk for {}, removed}}", + moduleKey, cacheFile, getDeviceProp(device).name); + } else { + if (modOut != nullptr) { CU_CHECK(cuModuleUnload(modOut)); } + AF_TRACE( + "{{{:<30} : cuModuleLoadData failed with content from {} for " + "{}, {}}}", + moduleKey, cacheFile, getDeviceProp(device).name, e.what()); + } removeFile(cacheFile); } return retVal; diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp index add7f58329..2f6d374db1 100644 --- a/src/backend/opencl/compile_module.cpp +++ b/src/backend/opencl/compile_module.cpp @@ -20,7 +20,10 @@ #include #include +#include +#include #include +#include #include #include #include @@ -37,9 +40,12 @@ using spdlog::logger; using std::begin; using std::end; +using std::ofstream; using std::ostringstream; using std::shared_ptr; using std::string; +using std::to_string; +using std::transform; using std::vector; using std::chrono::duration_cast; using std::chrono::high_resolution_clock; @@ -50,21 +56,30 @@ logger *getLogger() { return logger.get(); } -#define THROW_BUILD_LOG_EXCEPTION(PROG) \ - do { \ - string build_error; \ - build_error.reserve(4096); \ - auto devices = PROG.getInfo(); \ - for (auto &device : PROG.getInfo()) { \ - build_error += \ - format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n", \ - device.getInfo(), \ - PROG.getBuildInfo(device), \ - PROG.getBuildInfo(device)); \ - } \ - string info = getEnvVar("AF_OPENCL_SHOW_BUILD_INFO"); \ - if (!info.empty() && info != "0") puts(build_error.c_str()); \ - AF_ERROR(build_error, AF_ERR_INTERNAL); \ +string getProgramBuildLog(const Program &prog) { + string build_error(""); + try { + build_error.reserve(4096); + auto devices = prog.getInfo(); + for (auto &device : prog.getInfo()) { + build_error += + format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n", + device.getInfo(), + prog.getBuildInfo(device), + prog.getBuildInfo(device)); + } + } catch (const cl::Error &e) { + build_error = format("Failed to fetch build log: {}", e.what()); + } + return build_error; +} + +#define THROW_BUILD_LOG_EXCEPTION(PROG) \ + do { \ + string build_error = getProgramBuildLog(PROG); \ + string info = getEnvVar("AF_OPENCL_SHOW_BUILD_INFO"); \ + if (!info.empty() && info != "0") puts(build_error.c_str()); \ + AF_ERROR(build_error, AF_ERR_INTERNAL); \ } while (0) namespace opencl { @@ -119,6 +134,21 @@ Program buildProgram(const vector &kernelSources, } // namespace opencl +string getKernelCacheFilename(const int device, const string &key) { + auto &dev = opencl::getDevice(device); + + unsigned vendorId = dev.getInfo(); + auto devName = dev.getInfo(); + string infix = to_string(vendorId) + "_" + devName; + + transform(infix.begin(), infix.end(), infix.begin(), + [](unsigned char c) { return std::toupper(c); }); + std::replace(infix.begin(), infix.end(), ' ', '_'); + + return "KER" + key + "_CL_" + infix + "_AF_" + + to_string(AF_API_VERSION_CURRENT) + ".bin"; +} + namespace common { Module compileModule(const string &moduleKey, const vector &sources, @@ -131,6 +161,52 @@ Module compileModule(const string &moduleKey, const vector &sources, auto program = opencl::buildProgram(sources, options); auto compileEnd = high_resolution_clock::now(); +#ifdef AF_CACHE_KERNELS_TO_DISK + const int device = opencl::getActiveDeviceId(); + const string &cacheDirectory = getCacheDirectory(); + if (!cacheDirectory.empty()) { + const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR + + getKernelCacheFilename(device, moduleKey); + const string tempFile = + cacheDirectory + AF_PATH_SEPARATOR + makeTempFilename(); + try { + auto binaries = program.getInfo(); + + // TODO Handle cases where program objects are created from contexts + // having multiple devices + const size_t clbinSize = binaries[0].size(); + const char *clbin = + reinterpret_cast(binaries[0].data()); + const size_t clbinHash = deterministicHash(clbin, clbinSize); + + // write module hash and binary data to file + ofstream out(tempFile, std::ios::binary); + + out.write(reinterpret_cast(&clbinHash), + sizeof(clbinHash)); + out.write(reinterpret_cast(&clbinSize), + sizeof(clbinSize)); + out.write(static_cast(clbin), clbinSize); + out.close(); + + // try to rename temporary file into final cache file, if this fails + // this means another thread has finished compiling this kernel + // before the current thread. + if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); } + } catch (const cl::Error &e) { + AF_TRACE("{{{:<30} : Failed to fetch opencl binary for {}, {}}}", + moduleKey, + opencl::getDevice(device).getInfo(), + e.what()); + } catch (const std::ios_base::failure &e) { + AF_TRACE("{{{:<30} : Failed writing binary to {} for {}, {}}}", + moduleKey, cacheFile, + opencl::getDevice(device).getInfo(), + e.what()); + } + } +#endif + AF_TRACE("{{{:<30} : {{ compile:{:>5} ms, {{ {} }}, {} }}}}", moduleKey, duration_cast(compileEnd - compileBegin).count(), fmt::join(options, " "), @@ -141,10 +217,65 @@ Module compileModule(const string &moduleKey, const vector &sources, Module loadModuleFromDisk(const int device, const string &moduleKey, const bool isJIT) { - UNUSED(device); - UNUSED(moduleKey); - UNUSED(isJIT); - return {}; + const string &cacheDirectory = getCacheDirectory(); + if (cacheDirectory.empty()) return Module{}; + + auto &dev = opencl::getDevice(device); + const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR + + getKernelCacheFilename(device, moduleKey); + Program program; + Module retVal{}; + try { + std::ifstream in(cacheFile, std::ios::binary); + if (!in.is_open()) { + AF_ERROR("Unable to open binary cache file", AF_ERR_INTERNAL); + } + in.exceptions(std::ios::failbit | std::ios::badbit); + + // TODO Handle cases where program objects are created from contexts + // having multiple devices + size_t clbinHash = 0; + in.read(reinterpret_cast(&clbinHash), sizeof(clbinHash)); + size_t clbinSize = 0; + in.read(reinterpret_cast(&clbinSize), sizeof(clbinSize)); + vector clbin(clbinSize); + in.read(reinterpret_cast(clbin.data()), clbinSize); + in.close(); + + const size_t recomputedHash = + deterministicHash(clbin.data(), clbinSize); + if (recomputedHash != clbinHash) { + AF_ERROR("Binary on disk seems to be corrupted", AF_ERR_LOAD_SYM); + } + program = Program(opencl::getContext(), {dev}, {clbin}); + program.build(); + + AF_TRACE("{{{:<30} : loaded from {} for {} }}", moduleKey, cacheFile, + dev.getInfo()); + retVal.set(program); + } catch (const AfError &e) { + if (e.getError() == AF_ERR_LOAD_SYM) { + AF_TRACE( + "{{{:<30} : Corrupt binary({}) found on disk for {}, removed}}", + moduleKey, cacheFile, dev.getInfo()); + } else { + AF_TRACE("{{{:<30} : Unable to open {} for {}}}", moduleKey, + cacheFile, dev.getInfo()); + } + removeFile(cacheFile); + } catch (const std::ios_base::failure &e) { + AF_TRACE("{{{:<30} : IO failure while loading {} for {}; {}}}", + moduleKey, cacheFile, dev.getInfo(), e.what()); + removeFile(cacheFile); + } catch (const cl::Error &e) { + AF_TRACE( + "{{{:<30} : Loading OpenCL binary({}) failed for {}; {}, Build " + "Log: {}}}", + moduleKey, cacheFile, dev.getInfo(), e.what(), + getProgramBuildLog(program)); + removeFile(cacheFile); + } + return retVal; } Kernel getKernel(const Module &mod, const string &nameExpr,