From eef519700470bbe6ef9a61a7a840538324ebf161 Mon Sep 17 00:00:00 2001 From: Ryan Field Date: Mon, 12 Dec 2022 11:37:18 +0000 Subject: [PATCH 01/74] fix script and config locations --- CMakeLists.txt | 2 +- src/objects/config.cxx | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 10f2b51..3dbbe00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ ENDIF() # Define Project PROJECT( fdpapi - VERSION 0.2.6 + VERSION 0.2.7 DESCRIPTION "C++ API for the FAIR Data Pipeline" HOMEPAGE_URL "https://github.com/FAIRDataPipeline/cppDataPipeline" LANGUAGES CXX C diff --git a/src/objects/config.cxx b/src/objects/config.cxx index 8681b9b..ba3ed64 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -240,14 +240,14 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { // Remove the Write Data Store from config file path Json::Value config_storage_location_value_; - if(config_file_path_.string().find(meta_data_()["write_data_store"].as()) !=std::string::npos){ - config_storage_location_value_["path"] = config_file_path_.string().replace( - config_file_path_.string().find(meta_data_()["write_data_store"].as()), - sizeof(meta_data_()["write_data_store"].as()) - 1, ""); - } - else { - config_storage_location_value_["path"] = config_file_path_.string(); + config_storage_location_value_["path"] = config_file_path_.string(); + std::size_t ind = config_file_path_.string().find(meta_data_()["write_data_store"].as()); + if(ind != std::string::npos){ + config_storage_location_value_["path"] = config_storage_location_value_["path"].asString().erase( + ind, meta_data_()["write_data_store"].as().length() + ); } + config_storage_location_value_["path"] = remove_backslash_from_path(config_storage_location_value_["path"].asString()); config_storage_location_value_["path"] = API::remove_leading_forward_slash(config_storage_location_value_["path"].asString()); config_storage_location_value_["public"] = true; @@ -280,16 +280,15 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->config_obj_ = ApiObject::from_json( j_config_obj ); - Json::Value script_storage_location_value_; - - if(script_file_path_.string().find(meta_data_()["write_data_store"].as()) !=std::string::npos){ - script_storage_location_value_["path"] = script_file_path_.string().replace( - script_file_path_.string().find(meta_data_()["write_data_store"].as()), - sizeof(meta_data_()["write_data_store"].as()) - 1, ""); - } - else { - script_storage_location_value_["path"] = script_file_path_.string(); + Json::Value script_storage_location_value_; + script_storage_location_value_["path"] = script_file_path_.string(); + ind = script_file_path_.string().find(meta_data_()["write_data_store"].as()); + if(ind != std::string::npos){ + script_storage_location_value_["path"] = script_storage_location_value_["path"].asString().erase( + ind, meta_data_()["write_data_store"].as().length() + ); } + script_storage_location_value_["path"] = remove_backslash_from_path(script_storage_location_value_["path"].asString()); script_storage_location_value_["path"] = API::remove_leading_forward_slash(script_storage_location_value_["path"].asString()); script_storage_location_value_["hash"] = calculate_hash_from_file(script_file_path_); From 65084787fb887af6d8796c8a91aa177b8215fbfe Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 24 Jan 2023 11:00:53 +0000 Subject: [PATCH 02/74] Attempting Boost regex fix for gcc 4.8 --- external/boost_regex.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/external/boost_regex.cmake b/external/boost_regex.cmake index df5ab80..4ed253e 100644 --- a/external/boost_regex.cmake +++ b/external/boost_regex.cmake @@ -1,6 +1,7 @@ message(STATUS "[Boost Regex]") set(BOOST_ENABLE_CMAKE ON CACHE INTERNAL "") +set(BOOST_REGEX_STANDALONE ON CACHE INTERNAL "Get just regex from Boost libraries") find_package(Boost COMPONENTS regex) if(NOT Boost_FOUND) @@ -8,12 +9,10 @@ if(NOT Boost_FOUND) message(STATUS "\tBoost Regex Will be installed.") message(STATUS "\tURL: ${BRX_URL}") - set(BOOST_REGEX_STANDALONE ON CACHE INTERNAL "Get just regex from Boost libraries") FetchContent_Declare( BOOSTREGEX URL ${BRX_URL} ) FetchContent_MakeAvailable(BOOSTREGEX) - endif() From 8df36dad9307169da49bb1fed5014527e0ebe7df Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 24 Jan 2023 17:22:55 +0000 Subject: [PATCH 03/74] Replaced Boost::regex with RE2 regex library --- CMakeLists.txt | 2 +- external/boost_regex.cmake | 18 ------------------ external/re2.cmake | 17 +++++++++++++++++ include/fdp/objects/config.hxx | 2 +- include/fdp/objects/metadata.hxx | 2 +- include/fdp/registry/api.hxx | 2 +- src/CMakeLists.txt | 18 ++---------------- src/objects/config.cxx | 3 ++- src/objects/metadata.cxx | 10 +++++++--- src/registry/api.cxx | 31 ++++++++++++++++++++----------- test/CMakeLists.txt | 2 +- 11 files changed, 53 insertions(+), 54 deletions(-) delete mode 100644 external/boost_regex.cmake create mode 100644 external/re2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a6ae36..0df9444 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,7 @@ include(external/curl.cmake) include(external/yaml_cpp.cmake) include(external/toml11.cmake) include(external/ghc.cmake) -include(external/boost_regex.cmake) +include(external/re2.cmake) include(external/digestpp.cmake) # Define and install library diff --git a/external/boost_regex.cmake b/external/boost_regex.cmake deleted file mode 100644 index 4ed253e..0000000 --- a/external/boost_regex.cmake +++ /dev/null @@ -1,18 +0,0 @@ -message(STATUS "[Boost Regex]") - -set(BOOST_ENABLE_CMAKE ON CACHE INTERNAL "") -set(BOOST_REGEX_STANDALONE ON CACHE INTERNAL "Get just regex from Boost libraries") - -find_package(Boost COMPONENTS regex) -if(NOT Boost_FOUND) - set(BRX_URL "https://github.com/boostorg/regex/archive/refs/tags/boost-1.79.0.zip") - - message(STATUS "\tBoost Regex Will be installed.") - message(STATUS "\tURL: ${BRX_URL}") - - FetchContent_Declare( - BOOSTREGEX - URL ${BRX_URL} - ) - FetchContent_MakeAvailable(BOOSTREGEX) -endif() diff --git a/external/re2.cmake b/external/re2.cmake new file mode 100644 index 0000000..c850147 --- /dev/null +++ b/external/re2.cmake @@ -0,0 +1,17 @@ +message(STATUS "[RE2]") + +set(RE2_ENABLE_TESTING OFF CACHE INTERNAL "") + +set(RE2_URL "https://github.com/google/re2.git") +set(RE2_COMMIT "2022-12-01") + +message(STATUS "\tre2 (regex library) will be installed.") +message(STATUS "\tURL: ${RE2_URL}") +message(STATUS "\tCOMMIT: ${RE2_COMMIT}") + +FetchContent_Declare( + RE2 + GIT_REPOSITORY ${RE2_URL} + GIT_TAG ${RE2_COMMIT} +) +FetchContent_MakeAvailable(RE2) diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index c83491b..f70e68e 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index 5069ce7..d07f84d 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include "digestpp/digestpp.hpp" diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index fddea6f..25d8f7d 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 937c5cd..5f627ff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,33 +47,19 @@ target_include_directories( $ ) -if(NOT Boost_FOUND) - - target_include_directories(fdpapi PRIVATE ${boostregex_SOURCE_DIR}/include) - target_link_directories(fdpapi PRIVATE ${boostregex_BINARY_DIR}) - -else() - - target_link_libraries(fdpapi PRIVATE Boost::regex ) - -endif() - - # Dependencies target_link_libraries(fdpapi PRIVATE toml11::toml11) target_link_libraries(fdpapi PRIVATE digestpp::digestpp) target_link_libraries(fdpapi PRIVATE ${CURL_LIBRARIES}) target_link_libraries(fdpapi PRIVATE yaml-cpp) +target_link_libraries(fdpapi PRIVATE re2::re2) +target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) if(BUILD_SHARED_LIBS) target_link_libraries(fdpapi PRIVATE jsoncpp_lib) else() target_link_libraries(fdpapi PRIVATE jsoncpp_static) endif() -# ghc_filesystem is now not included at top level -target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) - - # Set rules for installing targets if(FDPAPI_WITH_INSTALL) message("Building With Install") diff --git a/src/objects/config.cxx b/src/objects/config.cxx index ba3ed64..29533d5 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -326,7 +326,8 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { Json::Value j_code_repo_root = api_->post("storage_root", repo_storage_root_value_, token_); this->code_repo_storage_root_ = ApiObject::from_json( j_code_repo_root ); - std::string repo_storage_path_ = boost::regex_replace(meta_data_()["remote_repo"].as(), boost::regex(repo_storage_root_value_["root"].asString()), ""); + std::string repo_storage_path_ = meta_data_()["remote_repo"].as(); + RE2::GlobalReplace(&repo_storage_path_, repo_storage_root_value_["root"].asString(), ""); Json::Value repo_storage_location_value_; repo_storage_location_value_["hash"] = meta_data_()["latest_commit"].as(); diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 6993855..6c23bdd 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -64,11 +64,15 @@ std::string current_time_stamp(bool file_name) { } std::string remove_local_from_root(const std::string &root){ - return boost::regex_replace(root, boost::regex(std::string("file:\\/\\/")), ""); + std::string result = root; + RE2::GlobalReplace(&result, "file:\\/\\/", ""); + return result; } std::string remove_backslash_from_path(const std::string &path){ - return boost::regex_replace(path, boost::regex(std::string("\\\\")), "/"); + std::string result = path; + RE2::GlobalReplace(&result, "\\\\", "/"); + return result; } bool file_exists( const std::string &Filename ) @@ -84,4 +88,4 @@ std::string read_token(const ghc::filesystem::path &token_path){ return key_str_; } -}; // namespace FairDataPipeline \ No newline at end of file +}; // namespace FairDataPipeline diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 6f2dd53..4970014 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -95,7 +95,8 @@ CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, Json::Value API::get_request(const ghc::filesystem::path &addr_path, long expected_response, std::string token) { - std::string addr_path_ = boost::regex_replace(addr_path.string(), boost::regex(std::string("\\\\")), "/"); + std::string addr_path_ = addr_path.string(); + RE2::GlobalReplace(&addr_path_, "\\\\", "/"); return get_request(addr_path_, expected_response); } @@ -167,6 +168,8 @@ std::string API::json_to_query_string(Json::Value &json_value) { std::string rtn = "?"; // Need to remove the api address from any values using regex std::string regex_string = "(" + url_root_ + ")([A-Za-z_]+)\\/([0-9]+)\\/"; + std::string match1, match2; + int match3; // Check the json value is not empty if (json_value.size() > 0) { // Iterate through the json keys @@ -179,18 +182,20 @@ std::string API::json_to_query_string(Json::Value &json_value) { i++) { // add the key and value to the return string after removing the api // address with regex - rtn += key + "=" + - boost::regex_replace(json_value.get(key, "")[i].asString(), - boost::regex(regex_string), "$3") + - "&"; + std::string str = json_value.get(key, "")[i].asString(); + if(RE2::FullMatch(str, regex_string, &match1, &match2, &match3)){ + str = std::to_string(match3); + } + rtn += key + "=" + str + "&"; } } else { // if it's not an array add the key and value to the return string after // removing the api address with regex - rtn += key + "=" + - boost::regex_replace(json_value.get(key, "").asString(), - boost::regex(regex_string), "$3") + - "&"; + std::string str = json_value.get(key, "").asString(); + if(RE2::FullMatch(str, regex_string, &match1, &match2, &match3)){ + str = std::to_string(match3); + } + rtn += key + "=" + str + "&"; } } } @@ -200,7 +205,9 @@ std::string API::json_to_query_string(Json::Value &json_value) { std::string API::escape_space(std::string &str) { // Using regex replace space with html character (%20) - return std::string(boost::regex_replace(str, boost::regex(" "), "%20")); + std::string result = str; + RE2::GlobalReplace(&result, " ", "%20"); + return result; } Json::Value API::post(std::string addr_path, Json::Value &post_data, @@ -222,7 +229,9 @@ Json::Value API::post_file_type(Json::Value &post_data, const std::string &token logger::get_logger()->error() << "Error: Post Data does not contain a file extension"; throw rest_apiquery_error("Failed to post file_type"); } - post_data["extension"] = boost::regex_replace(post_data["extension"].asString(), boost::regex("."), ""); + std::string extension = post_data["extension"].asString(); + RE2::GlobalReplace(&extension, ".", ""); + post_data["extension"] = extension; Json::Value _file_type_query; _file_type_query["extension"] = post_data["extension"]; Json::Value _file_type_exists = get_by_json_query("file_type", _file_type_query); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4ef7830..c4efdbf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -64,7 +64,7 @@ target_link_libraries(fdpapi-tests PRIVATE gtest gtest_main) target_link_libraries(fdpapi-tests PRIVATE toml11::toml11) target_link_libraries(fdpapi-tests PRIVATE digestpp) -target_link_libraries(fdpapi-tests PRIVATE boost_regex) +target_link_libraries(fdpapi-tests PRIVATE re2::re2) target_link_libraries(fdpapi-tests PRIVATE ${CURL_LIBRARIES}) target_link_libraries(fdpapi-tests PRIVATE yaml-cpp) if(BUILD_SHARED_LIBS) From 1bafb88aad03198708c3f06316479955cff51329 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 24 Jan 2023 17:37:51 +0000 Subject: [PATCH 04/74] Fixed RE2 building tests --- external/re2.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/re2.cmake b/external/re2.cmake index c850147..18fae29 100644 --- a/external/re2.cmake +++ b/external/re2.cmake @@ -1,6 +1,6 @@ message(STATUS "[RE2]") -set(RE2_ENABLE_TESTING OFF CACHE INTERNAL "") +set(RE2_BUILD_TESTING OFF CACHE INTERNAL "") set(RE2_URL "https://github.com/google/re2.git") set(RE2_COMMIT "2022-12-01") From 645e172c3a700eab147ff84203cfc5cd70ecda7b Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 11:04:30 +0000 Subject: [PATCH 05/74] Switch boost::regex with re2 in fdpapiConfig.cmake.in --- .gitignore | 1 + cmake_modules/fdpapiConfig.cmake.in | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 23c7e14..04b9bcf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ data_store/ venv/ test/data/temp/ Testing/ +CMakeFiles/ diff --git a/cmake_modules/fdpapiConfig.cmake.in b/cmake_modules/fdpapiConfig.cmake.in index 945f7d4..b95a475 100644 --- a/cmake_modules/fdpapiConfig.cmake.in +++ b/cmake_modules/fdpapiConfig.cmake.in @@ -8,7 +8,7 @@ find_package(digestpp) find_package(ghc_filesystem) find_package(jsoncpp) find_package(yaml-cpp) -find_package(Boost COMPONENTS regex QUIET) +find_package(re2) include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") From 9dcfbb62031437ae48f0379008b9e4b5529e127b Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 17:31:46 +0000 Subject: [PATCH 06/74] Added find_package alternatives for dependencies --- .gitignore | 1 + external/curl.cmake | 55 +++++++++++++++++++++++++++----------- external/digestpp.cmake | 58 +++++++++++++++++++++++++++++++---------- external/ghc.cmake | 50 ++++++++++++++++++++++++++++------- external/jsoncpp.cmake | 48 +++++++++++++++++++++++++++------- external/re2.cmake | 52 ++++++++++++++++++++++++++++-------- external/toml11.cmake | 48 ++++++++++++++++++++++++++++------ external/yaml_cpp.cmake | 53 +++++++++++++++++++++++++++++-------- src/CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- 10 files changed, 290 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 04b9bcf..339c4c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build/ +install/ .DS_STORE .vscode .bash_profile diff --git a/external/curl.cmake b/external/curl.cmake index 2faf1dd..1d33f48 100644 --- a/external/curl.cmake +++ b/external/curl.cmake @@ -1,23 +1,48 @@ -# Try and find CURL -FIND_PACKAGE( CURL QUIET ) -MESSAGE( STATUS "[Curl]" ) +message(STATUS "[Curl]") -# If CURL is not Found, Install it -IF(NOT CURL_FOUND) +option( + FDPAPI_FETCH_CURL_ONLY + "Don't use pre-installed curl. Recommended for testing only." + OFF +) - SET( CURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip" ) +option( + FDPAPI_FETCH_CURL_NEVER + "Only use pre-installed curl. Recommended for testing only." + OFF +) + +if(FDPAPI_FETCH_CURL_ONLY AND FDPAPI_FETCH_CURL_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_CURL_ONLY and FDPAPI_FETCH_CURL_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_CURL_ONLY) + find_package(CURL QUIET) +endif() + + +if(CURL_FOUND AND NOT FDPAPI_FETCH_CURL_ONLY) + message(STATUS "\tCURL found.") +elseif(NOT FDPAPI_FETCH_CURL_NEVER) + set(CURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip") + + message(STATUS "\tCURL not found.") + message(STATUS "\tInstalling from URL: ${CURL_URL}") - MESSAGE( STATUS "\tCURL Will be installed." ) - MESSAGE( STATUS "\tURL: ${CURL_URL}" ) - include(FetchContent) FetchContent_Declare( CURL URL ${CURL_URL} ) FetchContent_MakeAvailable(CURL) - SET(CURL_INCLUDE_DIRS = ${curl_SOURCE_DIR}/include) - SET(CURL_LIBRARIES libcurl) -ELSE() - MESSAGE( STATUS "\tInclude Directory: ${CURL_INCLUDE_DIRS}" ) - MESSAGE( STATUS "\tLibraries: ${CURL_LIBRARIES}" ) -ENDIF() \ No newline at end of file + + set(CURL_INCLUDE_DIRS = ${curl_SOURCE_DIR}/include) + set(CURL_LIBRARIES libcurl) +else() + message(FATAL_ERROR "\tCURL not found.") +endif() + +message(STATUS "\tInclude Directory: ${CURL_INCLUDE_DIRS}") +message(STATUS "\tLibraries: ${CURL_LIBRARIES}") diff --git a/external/digestpp.cmake b/external/digestpp.cmake index 91b1d51..b5ffd8a 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -1,12 +1,49 @@ -set(DIGESTCPP_URL "https://github.com/LiamPattinson/digestpp.git") -set(DIGESTCPP_COMMIT "6f6f134") +message(STATUS "[digestpp]" ) -message(STATUS "[DigestCPP]" ) +option( + FDPAPI_FETCH_DIGESTPP_ONLY + "Don't use pre-installed digestpp. Recommended for testing only." + OFF +) + +option( + FDPAPI_FETCH_DIGESTPP_NEVER + "Only use pre-installed digestpp. Recommended for testing only." + OFF +) + +if(FDPAPI_FETCH_DIGESTPP_ONLY AND FDPAPI_FETCH_DIGESTPP_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_DIGESTPP_ONLY and FDPAPI_FETCH_DIGESTPP_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_DIGESTPP_ONLY) + find_package(digestpp QUIET) +endif() + +if(digestpp_FOUND AND NOT FDPAPI_FETCH_DIGESTPP_ONLY) + message(STATUS "\tdigestpp found.") +elseif(NOT FDPAPI_FETCH_DIGESTPP_NEVER) + set(DIGESTPP_URL "https://github.com/LiamPattinson/digestpp.git") + set(DIGESTPP_COMMIT "6f6f134") + + message(STATUS "\tdigestpp not found.") + message(STATUS "\tdigestpp will be installed.") + message(STATUS "\t(Using a fork with a CMake build system)") + message(STATUS "\tURL: ${DIGESTPP_URL}") + message(STATUS "\tCOMMIT HASH: ${DIGESTPP_COMMIT}") -message(STATUS "\tDigestCpp Will be installed.") -message(STATUS "\t(Using a fork with a CMake build system)") -message(STATUS "\tURL: ${DIGESTCPP_URL}") -message(STATUS "\tCOMMIT HASH: ${DIGESTCPP_COMMIT}") + FetchContent_Declare( + DIGESTPP + GIT_REPOSITORY ${DIGESTPP_URL} + GIT_TAG ${DIGESTPP_COMMIT} + ) + FetchContent_MakeAvailable(DIGESTPP) +else() + message(FATAL_ERROR "\tdigestpp not found.") +endif() # windows.h will conflict with min functions in digestcpp # Because of macro definitions of min and max @@ -15,10 +52,3 @@ if(WIN32) add_definitions(-DNOMINMAX) add_definitions(-DNOGDI) endif() - -FetchContent_Declare( - DIGESTPP - GIT_REPOSITORY ${DIGESTCPP_URL} - GIT_TAG ${DIGESTCPP_COMMIT} -) -FetchContent_MakeAvailable(DIGESTPP) diff --git a/external/ghc.cmake b/external/ghc.cmake index e8bf803..d351b3c 100644 --- a/external/ghc.cmake +++ b/external/ghc.cmake @@ -1,12 +1,44 @@ -set(GHC_URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip") - -message( STATUS "[GHC]" ) -message( STATUS "\tGHC Will be installed." ) -message( STATUS "\tURL: ${GHC_URL}" ) +message(STATUS "[GHC]") set(GHC_FILESYSTEM_WITH_INSTALL ON CACHE INTERNAL "Create import targets for ghc-filesystem") -FetchContent_Declare( - GHC - URL ${GHC_URL} + +option( + FDPAPI_FETCH_GHC_ONLY + "Don't use pre-installed ghc-filesystem. Recommended for testing only." + OFF ) -FetchContent_MakeAvailable(GHC) + +option( + FDPAPI_FETCH_GHC_NEVER + "Only use pre-installed ghc-filesystem. Recommended for testing only." + OFF +) + +if(FDPAPI_FETCH_GHC_ONLY AND FDPAPI_FETCH_GHC_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_GHC_ONLY and FDPAPI_FETCH_GHC_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_GHC_ONLY) + find_package(ghc_filesystem QUIET) +endif() + +if(ghc_filesystem_FOUND AND NOT FDPAPI_FETCH_GHC_ONLY) + message(STATUS "\tGHC found.") +elseif(NOT FDPAPI_FETCH_GHC_NEVER) + set(GHC_URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip") + + message(STATUS "\tGHC not found.") + message(STATUS "\tGHC will be installed.") + message(STATUS "\tURL: ${GHC_URL}") + + FetchContent_Declare( + GHC + URL ${GHC_URL} + ) + FetchContent_MakeAvailable(GHC) +else() + message(FATAL_ERROR "\tGHC not found.") +endif() diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index 913ab33..0c707a9 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -1,14 +1,44 @@ -SET( JSONCPP_URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip" ) +message(STATUS "[JsonCPP]") -MESSAGE( STATUS "[JsonCPP]" ) -MESSAGE( STATUS "\tJsonCpp Will be installed." ) -MESSAGE( STATUS "\tURL: ${JSONCPP_URL}" ) +set(JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't build json-cpp tests") -SET (JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't build json-cpp tests") +option( + FDPAPI_FETCH_JSONCPP_ONLY + "Don't use pre-installed JsonCpp. Recommended for testing only." + OFF +) -FetchContent_Declare( - JsonCpp - URL ${JSONCPP_URL} +option( + FDPAPI_FETCH_JSONCPP_NEVER + "Only use pre-installed JsonCpp. Recommended for testing only." + OFF ) -FetchContent_MakeAvailable(JsonCpp) +if(FDPAPI_FETCH_JSONCPP_ONLY AND FDPAPI_FETCH_JSONCPP_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_JSONCPP_ONLY and FDPAPI_FETCH_JSONCPP_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_JSONCPP_ONLY) + find_package(jsoncpp QUIET) +endif() + +if(jsoncpp_FOUND AND NOT FDPAPI_FETCH_JSONCPP_ONLY) + message(STATUS "\tJsonCpp found.") +elseif(NOT FDPAPI_FETCH_JSONCPP_NEVER) + set(JSONCPP_URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip") + + message(STATUS "\tJsonCpp not found.") + message(STATUS "\tJsonCpp will be installed.") + message(STATUS "\tURL: ${JSONCPP_URL}") + + FetchContent_Declare( + JsonCpp + URL ${JSONCPP_URL} + ) + FetchContent_MakeAvailable(JsonCpp) +else() + message(FATAL_ERROR "\tJsonCpp not found.") +endif() diff --git a/external/re2.cmake b/external/re2.cmake index 18fae29..6ebb9b9 100644 --- a/external/re2.cmake +++ b/external/re2.cmake @@ -2,16 +2,46 @@ message(STATUS "[RE2]") set(RE2_BUILD_TESTING OFF CACHE INTERNAL "") -set(RE2_URL "https://github.com/google/re2.git") -set(RE2_COMMIT "2022-12-01") - -message(STATUS "\tre2 (regex library) will be installed.") -message(STATUS "\tURL: ${RE2_URL}") -message(STATUS "\tCOMMIT: ${RE2_COMMIT}") +option( + FDPAPI_FETCH_RE2_ONLY + "Don't use pre-installed RE2. Recommended for testing only." + OFF +) -FetchContent_Declare( - RE2 - GIT_REPOSITORY ${RE2_URL} - GIT_TAG ${RE2_COMMIT} +option( + FDPAPI_FETCH_RE2_NEVER + "Only use pre-installed RE2. Recommended for testing only." + OFF ) -FetchContent_MakeAvailable(RE2) + +if(FDPAPI_FETCH_RE2_ONLY AND FDPAPI_FETCH_RE2_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_RE2_ONLY and FDPAPI_FETCH_RE2_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_RE2_ONLY) + find_package(re2 QUIET) +endif() + +if(re2_FOUND AND NOT FDPAPI_FETCH_RE2_ONLY) + message(STATUS "\tRE2 found.") +elseif(NOT FDPAPI_FETCH_RE2_NEVER) + set(RE2_URL "https://github.com/google/re2.git") + set(RE2_COMMIT "2022-12-01") + + message(STATUS "\tRE2 not found.") + message(STATUS "\tRE2 will be installed.") + message(STATUS "\tURL: ${RE2_URL}") + message(STATUS "\tCOMMIT: ${RE2_COMMIT}") + + FetchContent_Declare( + RE2 + GIT_REPOSITORY ${RE2_URL} + GIT_TAG ${RE2_COMMIT} + ) + FetchContent_MakeAvailable(RE2) +else() + message(FATAL_ERROR "\tRE2 not found.") +endif() diff --git a/external/toml11.cmake b/external/toml11.cmake index 715134f..26baa6c 100644 --- a/external/toml11.cmake +++ b/external/toml11.cmake @@ -1,11 +1,43 @@ -SET( TOML11_URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" ) -MESSAGE( STATUS "[TOML 11]" ) +message(STATUS "[TOML 11]") -MESSAGE( STATUS "\tTOML 11 Will be installed." ) -MESSAGE( STATUS "\tURL: ${TOML11_URL}" ) +option( + FDPAPI_FETCH_TOML11_ONLY + "Don't use pre-installed TOML11. Recommended for testing only." + OFF +) -FetchContent_Declare( - toml11 - URL ${TOML11_URL} +option( + FDPAPI_FETCH_TOML11_NEVER + "Only use pre-installed TOML 11. Recommended for testing only." + OFF ) -FetchContent_MakeAvailable(toml11) + +if(FDPAPI_FETCH_TOML11_ONLY AND FDPAPI_FETCH_TOML11_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_TOML11_ONLY and FDPAPI_FETCH_TOML11_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_TOML11_ONLY) + find_package(toml11 QUIET) +endif() + + +if(toml11_FOUND AND NOT FDPAPI_FETCH_TOML11_ONLY) + message(STATUS "\tTOML11 found.") +elseif(NOT FDPAPI_FETCH_TOML11_NEVER) + set( TOML11_URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" ) + + message(STATUS "\tTOML 11 not found.") + message(STATUS "\tTOML 11 will be installed.") + message(STATUS "\tURL: ${TOML11_URL}") + + FetchContent_Declare( + toml11 + URL ${TOML11_URL} + ) + FetchContent_MakeAvailable(toml11) +else() + message(FATAL_ERROR "\tTOML 11 not found.") +endif() diff --git a/external/yaml_cpp.cmake b/external/yaml_cpp.cmake index 0d67887..9efe5b2 100644 --- a/external/yaml_cpp.cmake +++ b/external/yaml_cpp.cmake @@ -1,10 +1,4 @@ -set(YAML_CPP_GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git") -set(YAML_CPP_GIT_TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f") - message(STATUS "[YAML-cpp]") -message(STATUS "\tYAML-cpp Will be installed.") -message(STATUS "\tURL: ${YAML_CPP_GIT_REPOSITORY}") -message(STATUS "\tCOMMIT HASH: ${YAML_CPP_GIT_TAG}") set(YAML_CPP_BUILD_TESTS OFF CACHE INTERNAL "Build yaml-cpp tests") if(BUILD_SHARED_LIBS) @@ -12,9 +6,46 @@ if(BUILD_SHARED_LIBS) endif() set(YAML_CPP_INSTALL ON CACHE INTERNAL "Include export targets for installation") -FetchContent_Declare( - yaml-cpp - GIT_REPOSITORY ${YAML_CPP_GIT_REPOSITORY} - GIT_TAG ${YAML_CPP_GIT_TAG} +option( + FDPAPI_FETCH_YAMLCPP_ONLY + "Don't use pre-installed YAML-cpp. Recommended for testing only." + OFF +) + +option( + FDPAPI_FETCH_YAMLCPP_NEVER + "Only use pre-installed YAML-cpp. Recommended for testing only." + OFF ) -FetchContent_MakeAvailable(yaml-cpp) + +if(FDPAPI_FETCH_YAMLCPP_ONLY AND FDPAPI_FETCH_YAMLCPP_NEVER) + message( + FATAL_ERROR + "FDPAPI_FETCH_YAMLCPP_ONLY and FDPAPI_FETCH_YAMLCPP_NEVER are mutually exclusive" + ) +endif() + +if(NOT FDPAPI_FETCH_YAMLCPP_ONLY) + find_package(yaml-cpp QUIET) +endif() + +if(yaml-cpp_FOUND AND NOT FDPAPI_FETCH_YAMLCPP_ONLY) + message(STATUS "\tYAML-cpp found.") +elseif(NOT FDPAPI_FETCH_YAMLCPP_NEVER) + set(YAML_CPP_GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git") + set(YAML_CPP_GIT_TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f") + + message(STATUS "\tYAML-cpp not found.") + message(STATUS "\tYAML-cpp will be installed.") + message(STATUS "\tURL: ${YAML_CPP_GIT_REPOSITORY}") + message(STATUS "\tCOMMIT HASH: ${YAML_CPP_GIT_TAG}") + + FetchContent_Declare( + yaml-cpp + GIT_REPOSITORY ${YAML_CPP_GIT_REPOSITORY} + GIT_TAG ${YAML_CPP_GIT_TAG} + ) + FetchContent_MakeAvailable(yaml-cpp) +else() + message(FATAL_ERROR "\tYAML-cpp not found.") +endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5f627ff..cb5090b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,7 +62,7 @@ endif() # Set rules for installing targets if(FDPAPI_WITH_INSTALL) - message("Building With Install") + message(STATUS "Building install components") if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/lib ) set(CMAKE_INSTALL_BINDIR ${CMAKE_INSTALL_PREFIX}/bin ) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c4efdbf..712bcc3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -63,7 +63,7 @@ target_link_libraries(fdpapi-tests PRIVATE fdpapi::fdpapi) target_link_libraries(fdpapi-tests PRIVATE gtest gtest_main) target_link_libraries(fdpapi-tests PRIVATE toml11::toml11) -target_link_libraries(fdpapi-tests PRIVATE digestpp) +target_link_libraries(fdpapi-tests PRIVATE digestpp::digestpp) target_link_libraries(fdpapi-tests PRIVATE re2::re2) target_link_libraries(fdpapi-tests PRIVATE ${CURL_LIBRARIES}) target_link_libraries(fdpapi-tests PRIVATE yaml-cpp) From 557d9af0beed0896ffd7c7590032651fdb21b002 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 17:56:31 +0000 Subject: [PATCH 07/74] Update unit test workflows, add universal FETCH_ONLY/NEVER options --- .github/workflows/fdp_cpp_api.yaml | 28 +++++++++++++++++++++++++--- CMakeLists.txt | 5 +++++ external/curl.cmake | 6 +++--- external/digestpp.cmake | 6 +++--- external/ghc.cmake | 6 +++--- external/jsoncpp.cmake | 6 +++--- external/re2.cmake | 6 +++--- external/toml11.cmake | 6 +++--- external/yaml_cpp.cmake | 6 +++--- 9 files changed, 51 insertions(+), 24 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 108ae1d..5307d11 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -45,7 +45,7 @@ jobs: sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install - name: Fetch cache uses: actions/cache@v2.1.5 with: @@ -79,6 +79,19 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - name: Build from Install + run: | + cmake --build build --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake --build build_from_install + build-wrapper-linux-x86-64 --out-dir bw-outputs cmake --build build --target coverage + if [ $? -eq 0 ]; then + echo "Unit tests completed successfully" + exit 0 + else + echo "Unit tests failed" + exit 1 + fi Build_MacOS: name: Build MacOS runs-on: macOS-latest @@ -92,10 +105,15 @@ jobs: brew install curl - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install - name: Build Library run: | cmake --build build + - name: Build from Install + run: | + cmake --build build --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake --build build_from_install Build_Windows: name: Build Windows runs-on: windows-latest @@ -106,9 +124,13 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1 - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install - name: Compile FDP-Cpp-API run: cmake --build build --config=Release + - name: Build from Install + cmake --build build --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake --build build_from_install Build_GCC_4_8: name: Build GCC 4.8 runs-on: ubuntu-18.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0df9444..13f06e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,11 @@ option(BUILD_SHARED_LIBS "Build Static Libraries" OFF) option(FDPAPI_BUILD_TESTS "Build unit tests" OFF) option(FDPAPI_CODE_COVERAGE "Run GCov and LCov code coverage tools" OFF) option(FDPAPI_WITH_INSTALL "Allow project to be installable" ON) +option(FDPAPI_FETCH_ONLY "Don't use pre-installed dependencies" OFF) +option(FDPAPI_FETCH_NEVER "Only use pre-installed dependencies" OFF) +if(FDPAPI_FETCH_ONLY AND FDPAPI_FETCH_NEVER) + message(FATAL_ERROR "FDPAPI_FETCH_ONLY and FDPAPI_FETCH_NEVER are mutually exclusive") +endif() # Disable Building in Debug as HD5 get_property(isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) diff --git a/external/curl.cmake b/external/curl.cmake index 1d33f48..3c12321 100644 --- a/external/curl.cmake +++ b/external/curl.cmake @@ -19,14 +19,14 @@ if(FDPAPI_FETCH_CURL_ONLY AND FDPAPI_FETCH_CURL_NEVER) ) endif() -if(NOT FDPAPI_FETCH_CURL_ONLY) +if(NOT (FDPAPI_FETCH_CURL_ONLY OR FDPAPI_FETCH_ONLY)) find_package(CURL QUIET) endif() -if(CURL_FOUND AND NOT FDPAPI_FETCH_CURL_ONLY) +if(CURL_FOUND AND NOT (FDPAPI_FETCH_CURL_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tCURL found.") -elseif(NOT FDPAPI_FETCH_CURL_NEVER) +elseif(NOT (FDPAPI_FETCH_CURL_NEVER OR FDPAPI_FETCH_NEVER)) set(CURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip") message(STATUS "\tCURL not found.") diff --git a/external/digestpp.cmake b/external/digestpp.cmake index b5ffd8a..5f93c5d 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -19,13 +19,13 @@ if(FDPAPI_FETCH_DIGESTPP_ONLY AND FDPAPI_FETCH_DIGESTPP_NEVER) ) endif() -if(NOT FDPAPI_FETCH_DIGESTPP_ONLY) +if(NOT (FDPAPI_FETCH_DIGESTPP_ONLY OR FDPAPI_FETCH_ONLY)) find_package(digestpp QUIET) endif() -if(digestpp_FOUND AND NOT FDPAPI_FETCH_DIGESTPP_ONLY) +if(digestpp_FOUND AND NOT (FDPAPI_FETCH_DIGESTPP_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tdigestpp found.") -elseif(NOT FDPAPI_FETCH_DIGESTPP_NEVER) +elseif(NOT (FDPAPI_FETCH_DIGESTPP_NEVER OR FDPAPI_FETCH_NEVER)) set(DIGESTPP_URL "https://github.com/LiamPattinson/digestpp.git") set(DIGESTPP_COMMIT "6f6f134") diff --git a/external/ghc.cmake b/external/ghc.cmake index d351b3c..0c903e0 100644 --- a/external/ghc.cmake +++ b/external/ghc.cmake @@ -21,13 +21,13 @@ if(FDPAPI_FETCH_GHC_ONLY AND FDPAPI_FETCH_GHC_NEVER) ) endif() -if(NOT FDPAPI_FETCH_GHC_ONLY) +if(NOT (FDPAPI_FETCH_GHC_ONLY OR FDPAPI_FETCH_ONLY)) find_package(ghc_filesystem QUIET) endif() -if(ghc_filesystem_FOUND AND NOT FDPAPI_FETCH_GHC_ONLY) +if(ghc_filesystem_FOUND AND NOT (FDPAPI_FETCH_GHC_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tGHC found.") -elseif(NOT FDPAPI_FETCH_GHC_NEVER) +elseif(NOT (FDPAPI_FETCH_GHC_NEVER OR FDPAPI_FETCH_NEVER)) set(GHC_URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip") message(STATUS "\tGHC not found.") diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index 0c707a9..ea1caaf 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -21,13 +21,13 @@ if(FDPAPI_FETCH_JSONCPP_ONLY AND FDPAPI_FETCH_JSONCPP_NEVER) ) endif() -if(NOT FDPAPI_FETCH_JSONCPP_ONLY) +if(NOT (FDPAPI_FETCH_JSONCPP_ONLY OR FDPAPI_FETCH_ONLY)) find_package(jsoncpp QUIET) endif() -if(jsoncpp_FOUND AND NOT FDPAPI_FETCH_JSONCPP_ONLY) +if(jsoncpp_FOUND AND NOT (FDPAPI_FETCH_JSONCPP_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tJsonCpp found.") -elseif(NOT FDPAPI_FETCH_JSONCPP_NEVER) +elseif(NOT (FDPAPI_FETCH_JSONCPP_NEVER OR FDPAPI_FETCH_NEVER)) set(JSONCPP_URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip") message(STATUS "\tJsonCpp not found.") diff --git a/external/re2.cmake b/external/re2.cmake index 6ebb9b9..4b70979 100644 --- a/external/re2.cmake +++ b/external/re2.cmake @@ -21,13 +21,13 @@ if(FDPAPI_FETCH_RE2_ONLY AND FDPAPI_FETCH_RE2_NEVER) ) endif() -if(NOT FDPAPI_FETCH_RE2_ONLY) +if(NOT (FDPAPI_FETCH_RE2_ONLY OR FDPAPI_FETCH_ONLY)) find_package(re2 QUIET) endif() -if(re2_FOUND AND NOT FDPAPI_FETCH_RE2_ONLY) +if(re2_FOUND AND NOT (FDPAPI_FETCH_RE2_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tRE2 found.") -elseif(NOT FDPAPI_FETCH_RE2_NEVER) +elseif(NOT (FDPAPI_FETCH_RE2_NEVER OR FDPAPI_FETCH_NEVER)) set(RE2_URL "https://github.com/google/re2.git") set(RE2_COMMIT "2022-12-01") diff --git a/external/toml11.cmake b/external/toml11.cmake index 26baa6c..a4eb196 100644 --- a/external/toml11.cmake +++ b/external/toml11.cmake @@ -19,14 +19,14 @@ if(FDPAPI_FETCH_TOML11_ONLY AND FDPAPI_FETCH_TOML11_NEVER) ) endif() -if(NOT FDPAPI_FETCH_TOML11_ONLY) +if(NOT (FDPAPI_FETCH_TOML11_ONLY OR FDPAPI_FETCH_ONLY)) find_package(toml11 QUIET) endif() -if(toml11_FOUND AND NOT FDPAPI_FETCH_TOML11_ONLY) +if(toml11_FOUND AND NOT (FDPAPI_FETCH_TOML11_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tTOML11 found.") -elseif(NOT FDPAPI_FETCH_TOML11_NEVER) +elseif(NOT (FDPAPI_FETCH_TOML11_NEVER OR FDPAPI_FETCH_NEVER)) set( TOML11_URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" ) message(STATUS "\tTOML 11 not found.") diff --git a/external/yaml_cpp.cmake b/external/yaml_cpp.cmake index 9efe5b2..45f5837 100644 --- a/external/yaml_cpp.cmake +++ b/external/yaml_cpp.cmake @@ -25,13 +25,13 @@ if(FDPAPI_FETCH_YAMLCPP_ONLY AND FDPAPI_FETCH_YAMLCPP_NEVER) ) endif() -if(NOT FDPAPI_FETCH_YAMLCPP_ONLY) +if(NOT (FDPAPI_FETCH_YAMLCPP_ONLY OR FDPAPI_FETCH_ONLY)) find_package(yaml-cpp QUIET) endif() -if(yaml-cpp_FOUND AND NOT FDPAPI_FETCH_YAMLCPP_ONLY) +if(yaml-cpp_FOUND AND NOT (FDPAPI_FETCH_YAMLCPP_ONLY OR FDPAPI_FETCH_ONLY)) message(STATUS "\tYAML-cpp found.") -elseif(NOT FDPAPI_FETCH_YAMLCPP_NEVER) +elseif(NOT (FDPAPI_FETCH_YAMLCPP_NEVER OR FDPAPI_FETCH_NEVER)) set(YAML_CPP_GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git") set(YAML_CPP_GIT_TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f") From f76daa173afe7cd7b82299e380319784b757f390 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 18:25:31 +0000 Subject: [PATCH 08/74] Fix broken workflow --- .github/workflows/fdp_cpp_api.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 5307d11..f20cf15 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -128,6 +128,7 @@ jobs: - name: Compile FDP-Cpp-API run: cmake --build build --config=Release - name: Build from Install + run: | cmake --build build --target install cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON cmake --build build_from_install From 096f29b20fc2b99b343592bdbf6acbbd653ce9ff Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 18:38:27 +0000 Subject: [PATCH 09/74] Stop apt install libjsoncpp-dev in workflows The version of JsonCpp in the latest ubuntu repos is out of date, and does not work with fdpapi. --- .github/workflows/fdp_cpp_api.yaml | 2 +- .github/workflows/test_with_simple_model.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index f20cf15..1b08bb8 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -42,7 +42,7 @@ jobs: - name: Install Dependencies run: | sudo apt update - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - name: Configure Library run: | cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index bbfda34..6607011 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -35,7 +35,7 @@ jobs: virtualenvs-in-project: true - name: Install Dependencies run: | - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev - name: Build and run seirs example run: | cd ../simpleModel From 40df5e83eaa180c7b0e4d84982cd973dd4d9c7f1 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Fri, 3 Feb 2023 18:43:54 +0000 Subject: [PATCH 10/74] Stop apt install libjsoncpp-dev in GCC 4.8 workflow --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 1b08bb8..22b6089 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -149,7 +149,7 @@ jobs: - name: Install Dependencies run: | sudo apt update - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - name: Configure Library run: | cmake -Bbuild From f534f400a4f5fc09fea69796353d4bfddaa4e3c8 Mon Sep 17 00:00:00 2001 From: LiamPattinson Date: Mon, 6 Feb 2023 14:11:51 +0000 Subject: [PATCH 11/74] Better external CURL bundling, Windows workflows --- .github/workflows/fdp_cpp_api.yaml | 6 +++--- external/curl.cmake | 8 +------- src/CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 1b08bb8..9cd3a45 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -129,9 +129,9 @@ jobs: run: cmake --build build --config=Release - name: Build from Install run: | - cmake --build build --target install + cmake --build build --config=Release --target install cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON - cmake --build build_from_install + cmake --build --config=Release build_from_install Build_GCC_4_8: name: Build GCC 4.8 runs-on: ubuntu-18.04 @@ -149,7 +149,7 @@ jobs: - name: Install Dependencies run: | sudo apt update - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - name: Configure Library run: | cmake -Bbuild diff --git a/external/curl.cmake b/external/curl.cmake index 3c12321..2b06eed 100644 --- a/external/curl.cmake +++ b/external/curl.cmake @@ -37,12 +37,6 @@ elseif(NOT (FDPAPI_FETCH_CURL_NEVER OR FDPAPI_FETCH_NEVER)) URL ${CURL_URL} ) FetchContent_MakeAvailable(CURL) - - set(CURL_INCLUDE_DIRS = ${curl_SOURCE_DIR}/include) - set(CURL_LIBRARIES libcurl) else() message(FATAL_ERROR "\tCURL not found.") -endif() - -message(STATUS "\tInclude Directory: ${CURL_INCLUDE_DIRS}") -message(STATUS "\tLibraries: ${CURL_LIBRARIES}") +endif() \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb5090b..69b9e2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,7 +50,7 @@ target_include_directories( # Dependencies target_link_libraries(fdpapi PRIVATE toml11::toml11) target_link_libraries(fdpapi PRIVATE digestpp::digestpp) -target_link_libraries(fdpapi PRIVATE ${CURL_LIBRARIES}) +target_link_libraries(fdpapi PRIVATE CURL::libcurl) target_link_libraries(fdpapi PRIVATE yaml-cpp) target_link_libraries(fdpapi PRIVATE re2::re2) target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 712bcc3..32705e2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -65,7 +65,7 @@ target_link_libraries(fdpapi-tests PRIVATE gtest gtest_main) target_link_libraries(fdpapi-tests PRIVATE toml11::toml11) target_link_libraries(fdpapi-tests PRIVATE digestpp::digestpp) target_link_libraries(fdpapi-tests PRIVATE re2::re2) -target_link_libraries(fdpapi-tests PRIVATE ${CURL_LIBRARIES}) +target_link_libraries(fdpapi-tests PRIVATE CURL::libcurl) target_link_libraries(fdpapi-tests PRIVATE yaml-cpp) if(BUILD_SHARED_LIBS) target_link_libraries(fdpapi-tests PRIVATE jsoncpp_lib) From a8f2b701dfdf384231f9ef4e05516ea404bf5a93 Mon Sep 17 00:00:00 2001 From: LiamPattinson Date: Mon, 6 Feb 2023 14:22:54 +0000 Subject: [PATCH 12/74] Fix typo in Windows workflow --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 9cd3a45..9011ffb 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -131,7 +131,7 @@ jobs: run: | cmake --build build --config=Release --target install cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON - cmake --build --config=Release build_from_install + cmake --build build_from_install --config=Release Build_GCC_4_8: name: Build GCC 4.8 runs-on: ubuntu-18.04 From 1bc0ecf6d53d410469d5250885d44aa9ce41832d Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Mon, 6 Feb 2023 16:52:32 +0000 Subject: [PATCH 13/74] Add macro for finding/fetching packages --- .github/workflows/fdp_cpp_api.yaml | 6 +- CMakeLists.txt | 11 ++- cmake_modules/fdpapi_add_external.cmake | 101 ++++++++++++++++++++++++ external/curl.cmake | 44 +---------- external/digestpp.cmake | 50 ++---------- external/ghc.cmake | 46 +---------- external/jsoncpp.cmake | 46 +---------- external/re2.cmake | 50 ++---------- external/toml11.cmake | 46 +---------- external/yaml_cpp.cmake | 49 ++---------- 10 files changed, 141 insertions(+), 308 deletions(-) create mode 100644 cmake_modules/fdpapi_add_external.cmake diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 9011ffb..a27a6e7 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -82,7 +82,7 @@ jobs: - name: Build from Install run: | cmake --build build --target install - cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON cmake --build build_from_install build-wrapper-linux-x86-64 --out-dir bw-outputs cmake --build build --target coverage if [ $? -eq 0 ]; then @@ -112,7 +112,7 @@ jobs: - name: Build from Install run: | cmake --build build --target install - cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON cmake --build build_from_install Build_Windows: name: Build Windows @@ -130,7 +130,7 @@ jobs: - name: Build from Install run: | cmake --build build --config=Release --target install - cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_FETCH_NEVER=ON + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON cmake --build build_from_install --config=Release Build_GCC_4_8: name: Build GCC 4.8 diff --git a/CMakeLists.txt b/CMakeLists.txt index 13f06e3..e1aa5f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,10 +19,10 @@ option(BUILD_SHARED_LIBS "Build Static Libraries" OFF) option(FDPAPI_BUILD_TESTS "Build unit tests" OFF) option(FDPAPI_CODE_COVERAGE "Run GCov and LCov code coverage tools" OFF) option(FDPAPI_WITH_INSTALL "Allow project to be installable" ON) -option(FDPAPI_FETCH_ONLY "Don't use pre-installed dependencies" OFF) -option(FDPAPI_FETCH_NEVER "Only use pre-installed dependencies" OFF) -if(FDPAPI_FETCH_ONLY AND FDPAPI_FETCH_NEVER) - message(FATAL_ERROR "FDPAPI_FETCH_ONLY and FDPAPI_FETCH_NEVER are mutually exclusive") +option(FDPAPI_ALWAYS_FETCH "Don't use pre-installed dependencies, use FetchContent instead" OFF) +option(FDPAPI_NEVER_FETCH "Only use pre-installed dependencies, don't use FetchContent" OFF) +if(FDPAPI_ALWAYS_FETCH AND FDPAPI_NEVER_FETCH) + message(FATAL_ERROR "FDPAPI_ALWAYS_FETCH and FDPAPI_NEVER_FETCH are mutually exclusive") endif() # Disable Building in Debug as HD5 @@ -55,6 +55,9 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() +# Include macro for adding external packages +include(cmake_modules/fdpapi_add_external.cmake) + # Include external library files include(external/jsoncpp.cmake) include(external/curl.cmake) diff --git a/cmake_modules/fdpapi_add_external.cmake b/cmake_modules/fdpapi_add_external.cmake new file mode 100644 index 0000000..0335cf9 --- /dev/null +++ b/cmake_modules/fdpapi_add_external.cmake @@ -0,0 +1,101 @@ +# Macro for finding packages, or installing them from online repos using FetchContent +# +# Required args +# ------------- +# pkg +# Name of the package. This should be a simplified version of the full package name, +# ideally in all-caps. It does not need to match the package name that is passed +# to find_package() +# +# Optional args +# ------------- +# PKG_NAME pkg_name +# The 'true' package name, which will be passed to find_package(). If not set, +# defaults to ${pkg} +# URL url +# The URL from which to download the package if find_package() is not successful. +# Usually set to a Github .zip archive. +# REPO repo +# The URL of a Git repository to pull from. Ignored if URL is set. It is recommended +# to always set TAG alongside this, or else the package will be pulled from the +# latest commits to main/master, which may be unstable. +# TAG tag +# The Git tag to use from REPO. Can be a commit hash, release tag, etc. +# +# Environment vars +# ---------------- +# FDPAPI_ALWAYS_FETCH_${pkg} +# Set to ON to skip find_package() and always use FetchContent for this package. +# FDPAPI_NEVER_FETCH_${pkg} +# Set to ON to skip FetchContent for this package. If find_package() is not +# successful, the build will fail. + +macro(fdpapi_add_external pkg) + message(STATUS "[${pkg}]") + cmake_parse_arguments(FDPAPI_${pkg} "" "PKG_NAME;URL;REPO;TAG" "" ${ARGN}) + + # Set default PKG_NAME. This is passed to find_package + if(NOT DEFINED FDPAPI_${pkg}_PKG_NAME) + set(FDPAPI_${pkg}_PKG_NAME ${pkg}) + endif() + + # Set options to allow the user to control how they wish to use this package + option( + FDPAPI_NEVER_FETCH_${pkg} + "Only use pre-installed ${pkg}, don't use FetchContent." + OFF + ) + option( + FDPAPI_ALWAYS_FETCH_${pkg} + "Don't use pre-installed ${pkg}, always use FetchContent." + OFF + ) + + if(FDPAPI_ALWAYS_FETCH_${pkg} AND FDPAPI_NEVER_FETCH_${pkg}) + message( + FATAL_ERROR + "FDPAPI_ALWAYS_FETCH_${pkg} and FDPAPI_NEVER_FETCH_${pkg} are mutually exclusive" + ) + endif() + + # Try to find package, unless the user has requested not to + if(NOT (FDPAPI_ALWAYS_FETCH_${pkg} OR FDPAPI_ALWAYS_FETCH)) + find_package(${FDPAPI_${pkg}_PKG_NAME} QUIET) + endif() + + # If found, write a message and continue. Otherwise, try FetchContent. + if(${FDPAPI_${pkg}_PKG_NAME}_FOUND AND NOT (FDPAPI_ALWAYS_FETCH_${pkg} OR FDPAPI_ALWAYS_FETCH)) + message(STATUS "\t${pkg} found.") + elseif(NOT (FDPAPI_NEVER_FETCH_${pkg} OR FDPAPI_NEVER_FETCH)) + message(STATUS "\t${pkg} not found.") + + # Prefer to use URL. Otherwise, use REPO, and prefer to use TAG in that case. + if(DEFINED FDPAPI_${pkg}_URL) + message(STATUS "\tInstalling from ${FDPAPI_${pkg}_URL}.") + FetchContent_Declare( + FDPAPI_${pkg}_PKG_NAME + URL "${FDPAPI_${pkg}_URL}" + ) + elseif(DEFINED FDPAPI_${pkg}_REPO) + message(STATUS "\tInstalling from ${FDPAPI_${pkg}_REPO}.") + if(DEFINED FDPAPI_${pkg}_TAG) + message(STATUS "\tGit tag: ${FDPAPI_${pkg}_TAG}.") + FetchContent_Declare( + FDPAPI_${pkg}_PKG_NAME + GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" + GIT_TAG "${FDPAPI_${pkg}_TAG}" + ) + else() + FetchContent_Declare( + FDPAPI_${pkg}_PKG_NAME + GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" + ) + endif() + else() + message(FATAL_ERROR "\t${pkg} could not be installed.") + endif() + FetchContent_MakeAvailable(FDPAPI_${pkg}_PKG_NAME) + else() + message(FATAL_ERROR "\t${pkg} not found.") + endif() +endmacro() diff --git a/external/curl.cmake b/external/curl.cmake index 2b06eed..1e0cea0 100644 --- a/external/curl.cmake +++ b/external/curl.cmake @@ -1,42 +1,4 @@ -message(STATUS "[Curl]") - -option( - FDPAPI_FETCH_CURL_ONLY - "Don't use pre-installed curl. Recommended for testing only." - OFF +fdpapi_add_external( + "CURL" + URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip" ) - -option( - FDPAPI_FETCH_CURL_NEVER - "Only use pre-installed curl. Recommended for testing only." - OFF -) - -if(FDPAPI_FETCH_CURL_ONLY AND FDPAPI_FETCH_CURL_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_CURL_ONLY and FDPAPI_FETCH_CURL_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_CURL_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(CURL QUIET) -endif() - - -if(CURL_FOUND AND NOT (FDPAPI_FETCH_CURL_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tCURL found.") -elseif(NOT (FDPAPI_FETCH_CURL_NEVER OR FDPAPI_FETCH_NEVER)) - set(CURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip") - - message(STATUS "\tCURL not found.") - message(STATUS "\tInstalling from URL: ${CURL_URL}") - - FetchContent_Declare( - CURL - URL ${CURL_URL} - ) - FetchContent_MakeAvailable(CURL) -else() - message(FATAL_ERROR "\tCURL not found.") -endif() \ No newline at end of file diff --git a/external/digestpp.cmake b/external/digestpp.cmake index 5f93c5d..7762db2 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -1,50 +1,10 @@ -message(STATUS "[digestpp]" ) - -option( - FDPAPI_FETCH_DIGESTPP_ONLY - "Don't use pre-installed digestpp. Recommended for testing only." - OFF -) - -option( - FDPAPI_FETCH_DIGESTPP_NEVER - "Only use pre-installed digestpp. Recommended for testing only." - OFF +fdpapi_add_external( + "DIGESTPP" + PKG_NAME "digestpp" + REPO "https://github.com/LiamPattinson/digestpp.git" + TAG "6f6f134" ) -if(FDPAPI_FETCH_DIGESTPP_ONLY AND FDPAPI_FETCH_DIGESTPP_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_DIGESTPP_ONLY and FDPAPI_FETCH_DIGESTPP_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_DIGESTPP_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(digestpp QUIET) -endif() - -if(digestpp_FOUND AND NOT (FDPAPI_FETCH_DIGESTPP_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tdigestpp found.") -elseif(NOT (FDPAPI_FETCH_DIGESTPP_NEVER OR FDPAPI_FETCH_NEVER)) - set(DIGESTPP_URL "https://github.com/LiamPattinson/digestpp.git") - set(DIGESTPP_COMMIT "6f6f134") - - message(STATUS "\tdigestpp not found.") - message(STATUS "\tdigestpp will be installed.") - message(STATUS "\t(Using a fork with a CMake build system)") - message(STATUS "\tURL: ${DIGESTPP_URL}") - message(STATUS "\tCOMMIT HASH: ${DIGESTPP_COMMIT}") - - FetchContent_Declare( - DIGESTPP - GIT_REPOSITORY ${DIGESTPP_URL} - GIT_TAG ${DIGESTPP_COMMIT} - ) - FetchContent_MakeAvailable(DIGESTPP) -else() - message(FATAL_ERROR "\tdigestpp not found.") -endif() - # windows.h will conflict with min functions in digestcpp # Because of macro definitions of min and max # So tell the compiler to exclude min and max macros in windows diff --git a/external/ghc.cmake b/external/ghc.cmake index 0c903e0..c52d7c9 100644 --- a/external/ghc.cmake +++ b/external/ghc.cmake @@ -1,44 +1,6 @@ -message(STATUS "[GHC]") - set(GHC_FILESYSTEM_WITH_INSTALL ON CACHE INTERNAL "Create import targets for ghc-filesystem") - -option( - FDPAPI_FETCH_GHC_ONLY - "Don't use pre-installed ghc-filesystem. Recommended for testing only." - OFF +fdpapi_add_external( + "GHC" + URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip" + PKG_NAME "ghc_filesystem" ) - -option( - FDPAPI_FETCH_GHC_NEVER - "Only use pre-installed ghc-filesystem. Recommended for testing only." - OFF -) - -if(FDPAPI_FETCH_GHC_ONLY AND FDPAPI_FETCH_GHC_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_GHC_ONLY and FDPAPI_FETCH_GHC_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_GHC_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(ghc_filesystem QUIET) -endif() - -if(ghc_filesystem_FOUND AND NOT (FDPAPI_FETCH_GHC_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tGHC found.") -elseif(NOT (FDPAPI_FETCH_GHC_NEVER OR FDPAPI_FETCH_NEVER)) - set(GHC_URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip") - - message(STATUS "\tGHC not found.") - message(STATUS "\tGHC will be installed.") - message(STATUS "\tURL: ${GHC_URL}") - - FetchContent_Declare( - GHC - URL ${GHC_URL} - ) - FetchContent_MakeAvailable(GHC) -else() - message(FATAL_ERROR "\tGHC not found.") -endif() diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index ea1caaf..11c8e9c 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -1,44 +1,6 @@ -message(STATUS "[JsonCPP]") - set(JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't build json-cpp tests") - -option( - FDPAPI_FETCH_JSONCPP_ONLY - "Don't use pre-installed JsonCpp. Recommended for testing only." - OFF +fdpapi_add_external( + "JSONCPP" + URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip" + PKG_NAME "jsoncpp" ) - -option( - FDPAPI_FETCH_JSONCPP_NEVER - "Only use pre-installed JsonCpp. Recommended for testing only." - OFF -) - -if(FDPAPI_FETCH_JSONCPP_ONLY AND FDPAPI_FETCH_JSONCPP_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_JSONCPP_ONLY and FDPAPI_FETCH_JSONCPP_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_JSONCPP_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(jsoncpp QUIET) -endif() - -if(jsoncpp_FOUND AND NOT (FDPAPI_FETCH_JSONCPP_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tJsonCpp found.") -elseif(NOT (FDPAPI_FETCH_JSONCPP_NEVER OR FDPAPI_FETCH_NEVER)) - set(JSONCPP_URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip") - - message(STATUS "\tJsonCpp not found.") - message(STATUS "\tJsonCpp will be installed.") - message(STATUS "\tURL: ${JSONCPP_URL}") - - FetchContent_Declare( - JsonCpp - URL ${JSONCPP_URL} - ) - FetchContent_MakeAvailable(JsonCpp) -else() - message(FATAL_ERROR "\tJsonCpp not found.") -endif() diff --git a/external/re2.cmake b/external/re2.cmake index 4b70979..cf7f4c0 100644 --- a/external/re2.cmake +++ b/external/re2.cmake @@ -1,47 +1,7 @@ -message(STATUS "[RE2]") - set(RE2_BUILD_TESTING OFF CACHE INTERNAL "") - -option( - FDPAPI_FETCH_RE2_ONLY - "Don't use pre-installed RE2. Recommended for testing only." - OFF +fdpapi_add_external( + "RE2" + REPO "https://github.com/google/re2.git" + TAG "2022-12-01" + PKG_NAME "re2" ) - -option( - FDPAPI_FETCH_RE2_NEVER - "Only use pre-installed RE2. Recommended for testing only." - OFF -) - -if(FDPAPI_FETCH_RE2_ONLY AND FDPAPI_FETCH_RE2_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_RE2_ONLY and FDPAPI_FETCH_RE2_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_RE2_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(re2 QUIET) -endif() - -if(re2_FOUND AND NOT (FDPAPI_FETCH_RE2_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tRE2 found.") -elseif(NOT (FDPAPI_FETCH_RE2_NEVER OR FDPAPI_FETCH_NEVER)) - set(RE2_URL "https://github.com/google/re2.git") - set(RE2_COMMIT "2022-12-01") - - message(STATUS "\tRE2 not found.") - message(STATUS "\tRE2 will be installed.") - message(STATUS "\tURL: ${RE2_URL}") - message(STATUS "\tCOMMIT: ${RE2_COMMIT}") - - FetchContent_Declare( - RE2 - GIT_REPOSITORY ${RE2_URL} - GIT_TAG ${RE2_COMMIT} - ) - FetchContent_MakeAvailable(RE2) -else() - message(FATAL_ERROR "\tRE2 not found.") -endif() diff --git a/external/toml11.cmake b/external/toml11.cmake index a4eb196..cca2531 100644 --- a/external/toml11.cmake +++ b/external/toml11.cmake @@ -1,43 +1,5 @@ -message(STATUS "[TOML 11]") - -option( - FDPAPI_FETCH_TOML11_ONLY - "Don't use pre-installed TOML11. Recommended for testing only." - OFF +fdpapi_add_external( + "TOML11" + URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" + PKG_NAME "toml11" ) - -option( - FDPAPI_FETCH_TOML11_NEVER - "Only use pre-installed TOML 11. Recommended for testing only." - OFF -) - -if(FDPAPI_FETCH_TOML11_ONLY AND FDPAPI_FETCH_TOML11_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_TOML11_ONLY and FDPAPI_FETCH_TOML11_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_TOML11_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(toml11 QUIET) -endif() - - -if(toml11_FOUND AND NOT (FDPAPI_FETCH_TOML11_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tTOML11 found.") -elseif(NOT (FDPAPI_FETCH_TOML11_NEVER OR FDPAPI_FETCH_NEVER)) - set( TOML11_URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" ) - - message(STATUS "\tTOML 11 not found.") - message(STATUS "\tTOML 11 will be installed.") - message(STATUS "\tURL: ${TOML11_URL}") - - FetchContent_Declare( - toml11 - URL ${TOML11_URL} - ) - FetchContent_MakeAvailable(toml11) -else() - message(FATAL_ERROR "\tTOML 11 not found.") -endif() diff --git a/external/yaml_cpp.cmake b/external/yaml_cpp.cmake index 45f5837..51c3db3 100644 --- a/external/yaml_cpp.cmake +++ b/external/yaml_cpp.cmake @@ -1,51 +1,12 @@ -message(STATUS "[YAML-cpp]") - set(YAML_CPP_BUILD_TESTS OFF CACHE INTERNAL "Build yaml-cpp tests") if(BUILD_SHARED_LIBS) set(YAML_BUILD_SHARED_LIBS ON CACHE INTERNAL "Build .so for yaml-cpp") endif() set(YAML_CPP_INSTALL ON CACHE INTERNAL "Include export targets for installation") -option( - FDPAPI_FETCH_YAMLCPP_ONLY - "Don't use pre-installed YAML-cpp. Recommended for testing only." - OFF -) - -option( - FDPAPI_FETCH_YAMLCPP_NEVER - "Only use pre-installed YAML-cpp. Recommended for testing only." - OFF +fdpapi_add_external( + "YAMLCPP" + REPO "https://github.com/jbeder/yaml-cpp.git" + TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f" + PKG_NAME "yaml-cpp" ) - -if(FDPAPI_FETCH_YAMLCPP_ONLY AND FDPAPI_FETCH_YAMLCPP_NEVER) - message( - FATAL_ERROR - "FDPAPI_FETCH_YAMLCPP_ONLY and FDPAPI_FETCH_YAMLCPP_NEVER are mutually exclusive" - ) -endif() - -if(NOT (FDPAPI_FETCH_YAMLCPP_ONLY OR FDPAPI_FETCH_ONLY)) - find_package(yaml-cpp QUIET) -endif() - -if(yaml-cpp_FOUND AND NOT (FDPAPI_FETCH_YAMLCPP_ONLY OR FDPAPI_FETCH_ONLY)) - message(STATUS "\tYAML-cpp found.") -elseif(NOT (FDPAPI_FETCH_YAMLCPP_NEVER OR FDPAPI_FETCH_NEVER)) - set(YAML_CPP_GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git") - set(YAML_CPP_GIT_TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f") - - message(STATUS "\tYAML-cpp not found.") - message(STATUS "\tYAML-cpp will be installed.") - message(STATUS "\tURL: ${YAML_CPP_GIT_REPOSITORY}") - message(STATUS "\tCOMMIT HASH: ${YAML_CPP_GIT_TAG}") - - FetchContent_Declare( - yaml-cpp - GIT_REPOSITORY ${YAML_CPP_GIT_REPOSITORY} - GIT_TAG ${YAML_CPP_GIT_TAG} - ) - FetchContent_MakeAvailable(yaml-cpp) -else() - message(FATAL_ERROR "\tYAML-cpp not found.") -endif() From 9a48f3d11100d9f5f7cb9f9a605e92ac7df2779e Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Mon, 6 Feb 2023 17:13:49 +0000 Subject: [PATCH 14/74] Adjust FetchContent package names in add_external macro --- cmake_modules/fdpapi_add_external.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake_modules/fdpapi_add_external.cmake b/cmake_modules/fdpapi_add_external.cmake index 0335cf9..d83932f 100644 --- a/cmake_modules/fdpapi_add_external.cmake +++ b/cmake_modules/fdpapi_add_external.cmake @@ -73,7 +73,7 @@ macro(fdpapi_add_external pkg) if(DEFINED FDPAPI_${pkg}_URL) message(STATUS "\tInstalling from ${FDPAPI_${pkg}_URL}.") FetchContent_Declare( - FDPAPI_${pkg}_PKG_NAME + ${pkg} URL "${FDPAPI_${pkg}_URL}" ) elseif(DEFINED FDPAPI_${pkg}_REPO) @@ -81,20 +81,20 @@ macro(fdpapi_add_external pkg) if(DEFINED FDPAPI_${pkg}_TAG) message(STATUS "\tGit tag: ${FDPAPI_${pkg}_TAG}.") FetchContent_Declare( - FDPAPI_${pkg}_PKG_NAME + ${pkg} GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" GIT_TAG "${FDPAPI_${pkg}_TAG}" ) else() FetchContent_Declare( - FDPAPI_${pkg}_PKG_NAME + ${pkg} GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" ) endif() else() message(FATAL_ERROR "\t${pkg} could not be installed.") endif() - FetchContent_MakeAvailable(FDPAPI_${pkg}_PKG_NAME) + FetchContent_MakeAvailable(${pkg}) else() message(FATAL_ERROR "\t${pkg} not found.") endif() From b46ad41fa22a585bc4e6b7bb02bd80b0068f7a5f Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Mon, 6 Feb 2023 18:01:32 +0000 Subject: [PATCH 15/74] Add 'build from install' test using simple model TODO: This uses a branch of cppSimpleModel that allows the user to prevent the use of FetchContent with fdpapi. It would be preferable to merge that branch with main and correct this one before proceeding. --- .github/workflows/test_with_simple_model.yaml | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index 6607011..25cd0b2 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -11,13 +11,13 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - with: + with: python-version: "3.9" architecture: "x64" - name: Install graphviz run: | sudo apt update - sudo apt-get install graphviz + sudo apt-get install graphviz sudo apt-get install -y gnuplot - name: Install local registry run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main @@ -46,7 +46,7 @@ jobs: cmake --build build fair init --ci --local fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml if: startsWith(github.ref, 'refs/tags/') != true - name: Build and run seirs example on tagged release run: | @@ -58,5 +58,32 @@ jobs: cmake --build build fair init --ci --local fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml if: startsWith(github.ref, 'refs/tags/') + Build_simple_model_from_install: + name: Build Ubuntu Simple Model + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + - name: Build + run: | + cmake -B build -DCMAKE_INSTALL_PREFIX=install + cmake -B build --target install + - name: Install graphviz + run: | + sudo apt update + sudo apt-get install graphviz + sudo apt-get install -y gnuplot + - name: Checkout SimpleModel + uses: actions/checkout@v3 + with: + repository: FAIRDataPipeline/cppSimpleModel + ref: prevent_fetchcontent_option + path: simpleModel + - name: Build Simple Model + run: | + cd simpleModel + cmake -B build -DCMAKE_INSTALL_PREFIX=../install -DFDPAPI_NO_FETCHCONTENT=ON + cmake --build build From 59fdca76e0a4abed9a551d5f318ebc4c05ee9484 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Mon, 6 Feb 2023 18:14:45 +0000 Subject: [PATCH 16/74] Fix workflows typo --- .github/workflows/test_with_simple_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index 25cd0b2..c41dcde 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -70,7 +70,7 @@ jobs: - name: Build run: | cmake -B build -DCMAKE_INSTALL_PREFIX=install - cmake -B build --target install + cmake --build build --target install - name: Install graphviz run: | sudo apt update From 19e2142ffc3220c05c9e375d346306fb32c02e51 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Mon, 6 Feb 2023 18:24:07 +0000 Subject: [PATCH 17/74] Renamed simple model 'build from install' test --- .github/workflows/test_with_simple_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index c41dcde..aba96d1 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -61,7 +61,7 @@ jobs: fair run --local data/seirs_config.yaml if: startsWith(github.ref, 'refs/tags/') Build_simple_model_from_install: - name: Build Ubuntu Simple Model + name: Build Ubuntu simple model from install runs-on: ubuntu-latest strategy: fail-fast: false From 76ff654aca643a93580567b2c8c2d4df5de1c71e Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:21:54 +0000 Subject: [PATCH 18/74] Update fdp_cpp_api.yaml Remove cpp-yaml to avoid issues with building --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index a27a6e7..41f9d55 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -149,7 +149,7 @@ jobs: - name: Install Dependencies run: | sudo apt update - sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev gcovr + sudo apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | cmake -Bbuild From 0779dc4b1c539b3bf1a63588989abf0295610b3f Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 10:09:56 +0100 Subject: [PATCH 19/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 41f9d55..54661fd 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -1,6 +1,6 @@ name: FDP C++ API -on: [push] +on: [push, workflow_dispatch] jobs: Build_Ubuntu: From 946bc4aff1871755a5d0603a8d74c4bd741c28cb Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 10:10:17 +0100 Subject: [PATCH 20/74] Update test_with_simple_model.yaml --- .github/workflows/test_with_simple_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index aba96d1..9bd7fb2 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -1,6 +1,6 @@ name: C++ Test Simple Model -on: [push] +on: [push, workflow_dispatch] jobs: Test_Simple_Model: From 3f4bed5bdf9b8da9a349367287ea0e994406b631 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:34:11 +0100 Subject: [PATCH 21/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 54661fd..eeeda26 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -134,7 +134,7 @@ jobs: cmake --build build_from_install --config=Release Build_GCC_4_8: name: Build GCC 4.8 - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From 5e74f4eab1b6ebbb85c5e22e21da95602e70d2c9 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:46:59 +0100 Subject: [PATCH 22/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index eeeda26..a65a70b 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -141,11 +141,24 @@ jobs: with: python-version: "3.9" architecture: "x64" - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1 - with: - version: 4.8 - platform: x64 + - name: Setup GCC 4.8.5 + run: | + sudo dpkg --add-architecture i386 + sudo apt update + sudo apt upgrade + sudo apt-get install gcc-multilib libstdc++6:i386 + wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.bz2 --no-check-certificate + tar xf gcc-4.8.5.tar.bz2 + # cd gcc-4.8.5 + # ./contrib/download_prerequisites + # cd .. + sed -i -e 's/__attribute__/\/\/__attribute__/g' gcc-4.8.5/gcc/cp/cfns.h + sed -i 's/struct ucontext/ucontext_t/g' gcc-4.8.5/libgcc/config/i386/linux-unwind.h + mkdir xgcc-4.8.5 + pushd xgcc-4.8.5 + $PWD/../gcc-4.8.5/configure --enable-languages=c,c++ --prefix=/usr --enable-shared --enable-plugin --program-suffix=-4.8.5 + make MAKEINFO="makeinfo --force" -j + sudo make install -j - name: Install Dependencies run: | sudo apt update From 0f8c846d69c0b352f699c42c8e8a51d713d25688 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:53:17 +0100 Subject: [PATCH 23/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index a65a70b..21bcb7f 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -147,6 +147,7 @@ jobs: sudo apt update sudo apt upgrade sudo apt-get install gcc-multilib libstdc++6:i386 + sudo apt install make wget git gcc g++ lhasa libgmp-dev libmpfr-dev libmpc-dev flex bison gettext texinfo ncurses-dev autoconf rsync wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.bz2 --no-check-certificate tar xf gcc-4.8.5.tar.bz2 # cd gcc-4.8.5 From f77cb1487a1e7273b8ae328ce4a051eaea6ca081 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:12:35 +0100 Subject: [PATCH 24/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 21bcb7f..17fbe29 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -150,11 +150,13 @@ jobs: sudo apt install make wget git gcc g++ lhasa libgmp-dev libmpfr-dev libmpc-dev flex bison gettext texinfo ncurses-dev autoconf rsync wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.bz2 --no-check-certificate tar xf gcc-4.8.5.tar.bz2 - # cd gcc-4.8.5 - # ./contrib/download_prerequisites - # cd .. + cd gcc-4.8.5 + ./contrib/download_prerequisites + cd .. sed -i -e 's/__attribute__/\/\/__attribute__/g' gcc-4.8.5/gcc/cp/cfns.h sed -i 's/struct ucontext/ucontext_t/g' gcc-4.8.5/libgcc/config/i386/linux-unwind.h + sed -i '/#include /a #include ' gcc-4.8.5/libsanitizer/asan/asan_linux.cc + sed -i 's/__res_state \\*statp = (__res_state\\*)state\\;/struct __res_state \\*statp = (struct __res_state\\*)state\\;/g' gcc-4.8.5/libsanitizer/tsan/tsan_platform_linux.cc mkdir xgcc-4.8.5 pushd xgcc-4.8.5 $PWD/../gcc-4.8.5/configure --enable-languages=c,c++ --prefix=/usr --enable-shared --enable-plugin --program-suffix=-4.8.5 From a970e0621458d99801c4051cf7fad0cde3782d96 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:30:35 +0100 Subject: [PATCH 25/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 17fbe29..87e8b81 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -134,34 +134,20 @@ jobs: cmake --build build_from_install --config=Release Build_GCC_4_8: name: Build GCC 4.8 - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: ubuntu:18.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: "3.9" architecture: "x64" - - name: Setup GCC 4.8.5 + - name: Setup GCC 4.8 run: | - sudo dpkg --add-architecture i386 sudo apt update sudo apt upgrade - sudo apt-get install gcc-multilib libstdc++6:i386 - sudo apt install make wget git gcc g++ lhasa libgmp-dev libmpfr-dev libmpc-dev flex bison gettext texinfo ncurses-dev autoconf rsync - wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.5/gcc-4.8.5.tar.bz2 --no-check-certificate - tar xf gcc-4.8.5.tar.bz2 - cd gcc-4.8.5 - ./contrib/download_prerequisites - cd .. - sed -i -e 's/__attribute__/\/\/__attribute__/g' gcc-4.8.5/gcc/cp/cfns.h - sed -i 's/struct ucontext/ucontext_t/g' gcc-4.8.5/libgcc/config/i386/linux-unwind.h - sed -i '/#include /a #include ' gcc-4.8.5/libsanitizer/asan/asan_linux.cc - sed -i 's/__res_state \\*statp = (__res_state\\*)state\\;/struct __res_state \\*statp = (struct __res_state\\*)state\\;/g' gcc-4.8.5/libsanitizer/tsan/tsan_platform_linux.cc - mkdir xgcc-4.8.5 - pushd xgcc-4.8.5 - $PWD/../gcc-4.8.5/configure --enable-languages=c,c++ --prefix=/usr --enable-shared --enable-plugin --program-suffix=-4.8.5 - make MAKEINFO="makeinfo --force" -j - sudo make install -j + sudo apt install build-essentials + sudo apt install g++-4.8 - name: Install Dependencies run: | sudo apt update From 561b6a14cfd1e6f070a43e95ea5e2d7d6a206418 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:32:16 +0100 Subject: [PATCH 26/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 87e8b81..80768b3 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -144,14 +144,13 @@ jobs: architecture: "x64" - name: Setup GCC 4.8 run: | - sudo apt update - sudo apt upgrade - sudo apt install build-essentials - sudo apt install g++-4.8 + apt update + apt install build-essentials + apt install g++-4.8 - name: Install Dependencies run: | - sudo apt update - sudo apt install -y lcov curl libcurl4-openssl-dev gcovr + apt update + apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | cmake -Bbuild From cf801a9db108df5dca105786ac867e751c63a1e9 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:34:53 +0100 Subject: [PATCH 27/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 80768b3..e4b4bf5 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -145,8 +145,9 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essentials + apt install build-essential apt install g++-4.8 + gcc --version - name: Install Dependencies run: | apt update From 709748e5a8d16f1333fc9a9a84eafb911f0e0645 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:36:43 +0100 Subject: [PATCH 28/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index e4b4bf5..c24935e 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -145,13 +145,12 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essential - apt install g++-4.8 + apt install build-essential -y + apt install g++-4.8 -y gcc --version - name: Install Dependencies run: | - apt update - apt install -y lcov curl libcurl4-openssl-dev gcovr + apt install -y lcov curl libcurl4-openssl-dev gcovr -y - name: Configure Library run: | cmake -Bbuild From 55e26d15544c27e669fdd562c330f79b08349aa9 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:43:15 +0100 Subject: [PATCH 29/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index c24935e..421ad81 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -146,11 +146,15 @@ jobs: run: | apt update apt install build-essential -y - apt install g++-4.8 -y + apt install g++-4.8 gcc-4.8 -y + apt install cmake + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 30 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 20 gcc --version + g++ --version - name: Install Dependencies run: | - apt install -y lcov curl libcurl4-openssl-dev gcovr -y + apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | cmake -Bbuild From bd8fcbc4c574437e596a36864b281f3e0ade07ad Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:47:50 +0100 Subject: [PATCH 30/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 421ad81..642acd2 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -148,8 +148,10 @@ jobs: apt install build-essential -y apt install g++-4.8 gcc-4.8 -y apt install cmake - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 30 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 20 + rm /usr/bin/gcc + rm /usr/bin/g++ + ln -s /usr/bin/gcc-4.8 /usr/bin/gcc + ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - name: Install Dependencies From 058965987045b922b8d965aa139b0031c1d5285e Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:55:07 +0100 Subject: [PATCH 31/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 642acd2..1eca51e 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -147,7 +147,7 @@ jobs: apt update apt install build-essential -y apt install g++-4.8 gcc-4.8 -y - apt install cmake + apt install cmake -y rm /usr/bin/gcc rm /usr/bin/g++ ln -s /usr/bin/gcc-4.8 /usr/bin/gcc From 35443a60aa275af160976d05b8c52faecee6e455 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:03:55 +0100 Subject: [PATCH 32/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 1eca51e..58e64a8 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -135,9 +135,10 @@ jobs: Build_GCC_4_8: name: Build GCC 4.8 runs-on: ubuntu-latest - container: ubuntu:18.04 - steps: - - uses: actions/checkout@v2 + container: + ubuntu:18.04 + --user root + steps: - uses: actions/setup-python@v2 with: python-version: "3.9" @@ -154,6 +155,9 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version + apt intall git -y + git -v + - uses: actions/checkout@v3.0.2 - name: Install Dependencies run: | apt install -y lcov curl libcurl4-openssl-dev gcovr From 75533b05559df08029008c0523df703c6e3f847b Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:05:08 +0100 Subject: [PATCH 33/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 58e64a8..3367c70 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -136,8 +136,8 @@ jobs: name: Build GCC 4.8 runs-on: ubuntu-latest container: - ubuntu:18.04 - --user root + image: ubuntu:18.04 + options: --user root steps: - uses: actions/setup-python@v2 with: From a1d2dcf024af467139cae3f1211fbe51902e564c Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:06:40 +0100 Subject: [PATCH 34/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 3367c70..08aea1f 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -155,7 +155,7 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - apt intall git -y + apt install git -y git -v - uses: actions/checkout@v3.0.2 - name: Install Dependencies From e053edbe5d955a0dae66f5128c99f33778e35e25 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:08:58 +0100 Subject: [PATCH 35/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 08aea1f..182f51c 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -156,7 +156,7 @@ jobs: gcc --version g++ --version apt install git -y - git -v + git --version - uses: actions/checkout@v3.0.2 - name: Install Dependencies run: | From 0c33fb5ebc7d6ee12ab7426ea14d6e73d9644a77 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:20:12 +0100 Subject: [PATCH 36/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 182f51c..107f1d7 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -163,7 +163,9 @@ jobs: apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | + cd "$GITHUB_WORKSPACE" cmake -Bbuild - name: Build Library - run: | + run: + cd "$GITHUB_WORKSPACE" cmake --build build From 8b19e48934c29472c71d861bf36785c959bfebbd Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:36:04 +0100 Subject: [PATCH 37/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 107f1d7..0f35d8d 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -157,15 +157,17 @@ jobs: g++ --version apt install git -y git --version - - uses: actions/checkout@v3.0.2 + - uses: actions/checkout@v3 - name: Install Dependencies run: | apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | - cd "$GITHUB_WORKSPACE" + cd $GITHUB_WORKSPACE + pwd cmake -Bbuild - name: Build Library run: - cd "$GITHUB_WORKSPACE" + cd $GITHUB_WORKSPACE + pwd cmake --build build From 3f2445b3c0597a4bf84b27d8ec6d4c22262041de Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:40:22 +0100 Subject: [PATCH 38/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 0f35d8d..9301f06 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -157,6 +157,7 @@ jobs: g++ --version apt install git -y git --version + chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v3 - name: Install Dependencies run: | @@ -165,9 +166,11 @@ jobs: run: | cd $GITHUB_WORKSPACE pwd + ls cmake -Bbuild - name: Build Library run: cd $GITHUB_WORKSPACE + ls pwd cmake --build build From f64c4f9751d839f5b008fd8e70c76debf0f7ef14 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:45:50 +0100 Subject: [PATCH 39/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 9301f06..731c7bb 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -155,9 +155,6 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - apt install git -y - git --version - chown -R $USER:$USER $GITHUB_WORKSPACE - uses: actions/checkout@v3 - name: Install Dependencies run: | @@ -165,9 +162,8 @@ jobs: - name: Configure Library run: | cd $GITHUB_WORKSPACE - pwd ls - cmake -Bbuild + cmake -S . -B build - name: Build Library run: cd $GITHUB_WORKSPACE From 6ffad5ab87a8c8ff372d581e61f83a8739138f9d Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:47:45 +0100 Subject: [PATCH 40/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 731c7bb..cc5b8cd 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -162,6 +162,7 @@ jobs: - name: Configure Library run: | cd $GITHUB_WORKSPACE + mkdir build ls cmake -S . -B build - name: Build Library From dea3bba095ab99d8d284e449aff5627871b58971 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:52:23 +0100 Subject: [PATCH 41/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index cc5b8cd..245b1b1 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -164,10 +164,8 @@ jobs: cd $GITHUB_WORKSPACE mkdir build ls - cmake -S . -B build + cmake -S $GITHUB_WORKSPACE -B build - name: Build Library run: cd $GITHUB_WORKSPACE - ls - pwd - cmake --build build + cmake --build $GITHUB_WORKSPACE/build From 3a5fa994507c4706f1947892fc62202f8345e0f4 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:58:37 +0100 Subject: [PATCH 42/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 245b1b1..44d1516 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -161,11 +161,10 @@ jobs: apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | - cd $GITHUB_WORKSPACE - mkdir build ls - cmake -S $GITHUB_WORKSPACE -B build + cmake -S . -B build + working-directory: $GITHUB_WORKSPACE - name: Build Library run: - cd $GITHUB_WORKSPACE - cmake --build $GITHUB_WORKSPACE/build + cmake --build build + working-directory: $GITHUB_WORKSPACE From a182422c52015b015e1542158ad4008237a2675a Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:03:15 +0100 Subject: [PATCH 43/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 44d1516..2d70943 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -163,8 +163,8 @@ jobs: run: | ls cmake -S . -B build - working-directory: $GITHUB_WORKSPACE + working-directory: github.workspace - name: Build Library run: cmake --build build - working-directory: $GITHUB_WORKSPACE + working-directory: github.workspace From b459025430e5bd5ff2ef88b9045f285a1937f751 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:12:28 +0100 Subject: [PATCH 44/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 2d70943..0346037 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -148,13 +148,17 @@ jobs: apt update apt install build-essential -y apt install g++-4.8 gcc-4.8 -y - apt install cmake -y rm /usr/bin/gcc rm /usr/bin/g++ ln -s /usr/bin/gcc-4.8 /usr/bin/gcc ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' + apt-get update + sudo apt install cmake + cmake --version - uses: actions/checkout@v3 - name: Install Dependencies run: | From eac522662f4739e819c8221f30ea87e7a884306f Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:12:53 +0100 Subject: [PATCH 45/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 0346037..0345417 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -157,7 +157,7 @@ jobs: wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update - sudo apt install cmake + sudo apt install cmake -y cmake --version - uses: actions/checkout@v3 - name: Install Dependencies From fd54c4b51ed8f262ef49e560ff865516d9574fa7 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:14:23 +0100 Subject: [PATCH 46/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 0345417..2883abb 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -157,7 +157,7 @@ jobs: wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update - sudo apt install cmake -y + apt install cmake -y cmake --version - uses: actions/checkout@v3 - name: Install Dependencies From 47d45249a986dbc9fa697f02787071d378024d82 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:16:36 +0100 Subject: [PATCH 47/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 2883abb..2ab2324 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -154,7 +154,7 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update apt install cmake -y From b82a37874d06a875ad7a9b4cd6a569241a7995c4 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:23:30 +0100 Subject: [PATCH 48/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 2ab2324..da29ea3 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -154,6 +154,7 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version + apt-get install ca-certificates wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update From 73b3e8ce03931cd44a239186377d89aa3371658f Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:26:45 +0100 Subject: [PATCH 49/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index da29ea3..14a1e86 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -154,7 +154,7 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - apt-get install ca-certificates + apt-get install ca-certificates -y wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update From e219ae9950e36c3c23794db73e9097396a29e11b Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:31:10 +0100 Subject: [PATCH 50/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 14a1e86..4c05da0 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -155,7 +155,7 @@ jobs: gcc --version g++ --version apt-get install ca-certificates -y - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - + wget --no-check-certificate -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' apt-get update apt install cmake -y From ad3446104bb01ddc80076cd8bdb1c469cf930148 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:36:59 +0100 Subject: [PATCH 51/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 4c05da0..a1e6057 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -7,7 +7,7 @@ jobs: name: Build Ubuntu runs-on: ubuntu-latest strategy: - fail-fast: false + fail-fast: true steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -154,9 +154,9 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - apt-get install ca-certificates -y - wget --no-check-certificate -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' + apt-get install ca-certificates gpg -y + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null apt-get update apt install cmake -y cmake --version From 60176ea354db02b0bddef9458b7da7bb3c95c2f4 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:44:28 +0100 Subject: [PATCH 52/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index a1e6057..b5492fd 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -7,7 +7,7 @@ jobs: name: Build Ubuntu runs-on: ubuntu-latest strategy: - fail-fast: true + fail-fast: false steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -146,7 +146,7 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essential -y + apt install build-essential libssl-dev -y apt install g++-4.8 gcc-4.8 -y rm /usr/bin/gcc rm /usr/bin/g++ @@ -154,11 +154,14 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - apt-get install ca-certificates gpg -y - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null - apt-get update - apt install cmake -y + wget https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0.tar.gz + tar -zvxf cmake-3.20.0.tar.gz + cd cmake-3.20.0 + ./bootstrap + make -j8 + apt-get install checkinstall -y + hash -r + checkinstall --pkgname=cmake --pkgversion="3.20-custom" --default cmake --version - uses: actions/checkout@v3 - name: Install Dependencies From 720ab871bac27df719e64d3f0181bce32891a23c Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:46:36 +0100 Subject: [PATCH 53/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index b5492fd..e811f35 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -146,7 +146,7 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essential libssl-dev -y + apt install build-essential libssl-dev cmake -y apt install g++-4.8 gcc-4.8 -y rm /usr/bin/gcc rm /usr/bin/g++ @@ -154,14 +154,6 @@ jobs: ln -s /usr/bin/g++-4.8 /usr/bin/g++ gcc --version g++ --version - wget https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0.tar.gz - tar -zvxf cmake-3.20.0.tar.gz - cd cmake-3.20.0 - ./bootstrap - make -j8 - apt-get install checkinstall -y - hash -r - checkinstall --pkgname=cmake --pkgversion="3.20-custom" --default cmake --version - uses: actions/checkout@v3 - name: Install Dependencies From d0206a5741835a8e7e222153d1f3017802a89604 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:53:59 +0100 Subject: [PATCH 54/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index e811f35..60d490d 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -146,7 +146,11 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essential libssl-dev cmake -y + apt install build-essential libssl-dev -y + git clone https://github.com/Kitware/CMake/ + cd CMake + ./bootstrap && make && sudo make install + cmake --version apt install g++-4.8 gcc-4.8 -y rm /usr/bin/gcc rm /usr/bin/g++ From 2bfccbf13759ecc6a888d953eb1c9f8ca68e9a49 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:56:03 +0100 Subject: [PATCH 55/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 60d490d..7b43e3f 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -146,7 +146,7 @@ jobs: - name: Setup GCC 4.8 run: | apt update - apt install build-essential libssl-dev -y + apt install build-essential libssl-dev git -y git clone https://github.com/Kitware/CMake/ cd CMake ./bootstrap && make && sudo make install From b47c7165b5183924b971f2f342cb2285ac48d1e4 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 17:02:25 +0100 Subject: [PATCH 56/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 7b43e3f..6de9e52 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -165,10 +165,10 @@ jobs: apt install -y lcov curl libcurl4-openssl-dev gcovr - name: Configure Library run: | + cd $GITHUB_WORKSPACE ls - cmake -S . -B build - working-directory: github.workspace + cmake -Bbuild - name: Build Library - run: + run: | + cd $GITHUB_WORKSPACE cmake --build build - working-directory: github.workspace From 6cad4ea3825c06fca48d61ce08e4993276567cd0 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:34:12 +0100 Subject: [PATCH 57/74] Update fdp_cpp_api.yaml --- .github/workflows/fdp_cpp_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 6de9e52..1aaaae1 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -149,7 +149,7 @@ jobs: apt install build-essential libssl-dev git -y git clone https://github.com/Kitware/CMake/ cd CMake - ./bootstrap && make && sudo make install + ./bootstrap && make && make install cmake --version apt install g++-4.8 gcc-4.8 -y rm /usr/bin/gcc From 08bcdeb83d5cdff9045d0369bed4a2e1eede8fa3 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 18 Apr 2023 17:43:51 +0100 Subject: [PATCH 58/74] Add C API Tests required --- include/fdp/fdp.h | 146 ++++++++++++++++++++++++++++ src/CMakeLists.txt | 2 + src/fdp_c_api.cxx | 233 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 381 insertions(+) create mode 100644 include/fdp/fdp.h create mode 100644 src/fdp_c_api.cxx diff --git a/include/fdp/fdp.h b/include/fdp/fdp.h new file mode 100644 index 0000000..f9b1142 --- /dev/null +++ b/include/fdp/fdp.h @@ -0,0 +1,146 @@ +#ifndef __FDP_C_API__ +#define __FDP_C_API__ + + +#ifdef __cplusplus +extern "C" { +#endif + + +/** + * @brief Enumeration used to denote different error types. + * + * The underlying C++ API will raise a number of different exception types. These map to + * integer error codes for C compatibility. + */ +enum FDP_ERR_T { + FDP_ERR_NONE = 0, + FDP_ERR_CONFIG_PARSE = 1, + FDP_ERR_REST_API_QUERY = 2, + FDP_ERR_JSON_PARSE = 3, + FDP_ERR_VALIDATION = 4, + FDP_ERR_SYNC = 5, + FDP_ERR_WRITE = 6, + FDP_ERR_TOML = 6, + FDP_ERR_OTHER = 7 +}; +typedef enum FDP_ERR_T FDP_ERR_T; + + +/** + * @brief Initialise the pipeline. + * + * Should be called once before any calls to fdp_link_read or fdp_link_write. If called + * more than once, returns FDP_ERR_OTHER. + * + * @param config_file_path Path to the `config.yaml` file for this FDP run. Should + * be at the location `${FDP_CONFIG_DIR}/config.yaml`. + * @param script_file_path Path to the script which initiates this FDP run. Should + * be at the location `${FDP_CONFIG_DIR}/script.sh` (or + * `${FDP_CONFIG_DIR}/script.bat` on Windows). + * @param token Token used to connect to FDP registry. May be set to `NULL`. + * + * @return Error code. + */ +FDP_ERR_T fdp_init( + const char *config_file_path, + const char *script_file_path, + const char *token +); + + +/** + * @brief Finalise the pipeline. + * + * Must be called after a call to fdp_init. + * + * Record all data products and meta data to the registry. Update the code run with all + * appropriate meta data. + * + * @return Error code. + */ +FDP_ERR_T fdp_finalise(); + + +/** + * @brief Set a path to a given data product while recording it's meta data for the + * code run. + * + * Must be called after fdp_init and before fdp_finalise. + * + * @param data_product Path to the input file. + * @param data_store_path Path to the assigned data store location. The user should + * allocate sufficient memory beforehand. + * @return Error code + */ +FDP_ERR_T fdp_link_read(const char *data_product, char *data_store_path); + + +/** + * @brief Set a path to a given data product while recording it's meta data for the + * code run. + * + * Must be called after fdp_init and before fdp_finalise. + * + * @param data_product Path to the output file. + * @param data_store_path Path to the assigned data store location. The user should + * allocate sufficient memory beforehand. + * @return Error code + */ +FDP_ERR_T fdp_link_write(const char *data_product, char *data_store_path); + + +/** + * @brief Enumeration used to denote the different levels of logging. + * + * Each level of logging includes all log levels greater than it, so setting the log + * level to `DEBUG` will include all log types except `TRACE`. These correspond to + * the C++ logging levels `FairDataPipeline::logging::LOG_LEVEL`. + */ +enum FDP_LOG_LEVEL { + FDP_LOG_TRACE = 0, + FDP_LOG_DEBUG = 1, + FDP_LOG_INFO = 2, + FDP_LOG_WARN = 3, + FDP_LOG_ERROR = 4, + FDP_LOG_CRITICAL = 5, + FDP_LOG_OFF = 6 +}; +typedef enum FDP_LOG_LEVEL FDP_LOG_LEVEL; + + +/** + * @brief Set the log level. Must call `fdp_init` first. + * + * @param log_level + */ +void fdp_set_log_level(FDP_LOG_LEVEL log_level); + + +/** + * @brief Get the current log level. Must call fdp_init first. + * + * @return Log level + */ +FDP_LOG_LEVEL fdp_get_log_level(); + + +/** + * @brief Write a message to the log. This will be passed to the C++ logger, + * `FairDataPipeline::logger::get_logger()->level() << msg`, where `level` + * is one of `trace`, `debug`, `info`, `warn`, `error`, or `critical`. + * + * @param log_level The type of log message to write, e.g. FDP_LOG_INFO, FDP_LOG_ERROR. + * @param msg The message to be written to log. + * + * @return Error code. 1 if logging unsuccessful, 0 otherwise. + */ +int fdp_log(FDP_LOG_LEVEL log_level, const char* msg); + + +#ifdef __cplusplus +} // close extern "C" +#endif + + +#endif // __FDP_C_API__ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 69b9e2b..ab35efa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ set(FDPAPI_SOURCE_FILES ../include/fdp/fdp.hxx + ../include/fdp/fdp.h ../include/fdp/exceptions.hxx ../include/fdp/objects/api_object.hxx ../include/fdp/objects/config.hxx @@ -12,6 +13,7 @@ set(FDPAPI_SOURCE_FILES ../include/fdp/utilities/logging.hxx ../include/fdp/utilities/semver.hxx ./fdp.cxx + ./fdp_c_api.cxx ./objects/api_object.cxx ./objects/config.cxx ./objects/distribution.cxx diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx new file mode 100644 index 0000000..74111a3 --- /dev/null +++ b/src/fdp_c_api.cxx @@ -0,0 +1,233 @@ +#include +#include +#include +#include + +#include "fdp/fdp.h" +#include "fdp/fdp.hxx" +#include "fdp/exceptions.hxx" +#include "fdp/utilities/logging.hxx" + + +namespace FDP = FairDataPipeline; + +/** + * @brief Utility method, calls exception-raising function and returns error codes + * + * Used to permit calls to C++ functions that may raise exceptions within a C + * environment. If any exceptions are thrown, these are caught and converted to an error + * code using the enum type FDP_ERR_T. + * + * Many functions in the C++ API operate on a shared pointer to a + * FairDataPipeline::DataPipeline object. To call a function on this shared pointer + * using exception_to_err_code, use a lambda that captures the shared pointer. + * + * @param function The C++ function to call. Must not have a void return type. + * @param ret Parameter in which to store the result of the function call. + * @param args Args to pass to the function. + * + * @return Error code + * + * @see exception_to_err_code_void + */ +template +FDP_ERR_T exception_to_err_code(Function&& function, Return& ret, Args&&... args){ + try{ + ret = std::forward(function)(std::forward(args)...); + return FDP_ERR_NONE; + } + catch(const FDP::config_parsing_error&){ + return FDP_ERR_CONFIG_PARSE; + } + catch(const FDP::rest_apiquery_error&){ + return FDP_ERR_REST_API_QUERY; + } + catch(const FDP::json_parse_error&){ + return FDP_ERR_JSON_PARSE; + } + catch(const FDP::validation_error&){ + return FDP_ERR_VALIDATION; + } + catch(const FDP::sync_error&){ + return FDP_ERR_SYNC; + } + catch(const FDP::write_error&){ + return FDP_ERR_WRITE; + } + catch(const FDP::toml_error&){ + return FDP_ERR_TOML; + } + catch(...){ + return FDP_ERR_OTHER; + } +} + +/** + * @brief Companion to exception_to_err_code for non-returning functions. + */ +template +FDP_ERR_T exception_to_err_code_void(Function&& function, Args&&... args){ + int dummy; + return exception_to_err_code( + [&function](Args&&... args) -> int { + std::forward(function)(std::forward(args)...); + return 0; + }, + dummy, + std::forward(args)... + ); +} + + +// ================= +// init and finalise +// ================= + + +/** + * @brief Global instance of FairDataPipeline::DataPipeline + * + * This should be initialised by fdp_init and deleted by fdp_finalise. The functions + * fdp_link_read and fdp_link_write make use of this, so will have undefined behaviour + * if called before init and after finalise. + */ +FDP::DataPipeline::sptr _datapipeline; + + +FDP_ERR_T fdp_init( + const char *config_file_path, + const char *script_file_path, + const char *token +){ + std::string token_str = (token == nullptr ? "" : token); + + if(_datapipeline == nullptr){ + return exception_to_err_code( + FDP::DataPipeline::construct, + _datapipeline, + std::string(config_file_path), + std::string(script_file_path), + token_str + ); + } + return FDP_ERR_OTHER; +} + + +FDP_ERR_T fdp_finalise(){ + if(_datapipeline == nullptr) return FDP_ERR_OTHER; + return exception_to_err_code_void( + [=](){ + _datapipeline->finalise(); + } + ); +} + + +template +FDP_ERR_T _fdp_link(LinkFunction&& link_function, const char* path, char* output){ + if(_datapipeline == nullptr) return FDP_ERR_OTHER; + std::string input_path = path; + std::string output_path; + FDP_ERR_T err = exception_to_err_code( + std::forward(link_function), + output_path, + input_path + ); + if(err){ + return err; + } + strcpy(output, output_path.c_str()); + return FDP_ERR_NONE; +} + + +FDP_ERR_T fdp_link_read(const char* path, char* output){ + return _fdp_link( + [=](std::string& path) -> std::string { + return _datapipeline->link_read(path); + }, + path, + output + ); +} + + +FDP_ERR_T fdp_link_write(const char* path, char* output){ + return _fdp_link( + [=](std::string& path) -> std::string { + return _datapipeline->link_write(path); + }, + path, + output + ); +} + +// ======= +// logging +// ======= + + +/** + * @brief Map converting C API logging enums to the C++ API + */ +std::map to_cpp_enum = { + {FDP_LOG_TRACE, FDP::logging::TRACE}, + {FDP_LOG_DEBUG, FDP::logging::DEBUG}, + {FDP_LOG_INFO, FDP::logging::INFO}, + {FDP_LOG_WARN, FDP::logging::WARN}, + {FDP_LOG_ERROR, FDP::logging::ERROR}, + {FDP_LOG_CRITICAL, FDP::logging::CRITICAL}, + {FDP_LOG_OFF, FDP::logging::OFF} +}; + + +/** + * @brief Map converting C++ API logging enums to the C API + */ +std::map to_c_enum = { + {FDP::logging::TRACE, FDP_LOG_TRACE}, + {FDP::logging::DEBUG, FDP_LOG_DEBUG}, + {FDP::logging::INFO, FDP_LOG_INFO}, + {FDP::logging::WARN, FDP_LOG_WARN}, + {FDP::logging::ERROR, FDP_LOG_ERROR}, + {FDP::logging::CRITICAL, FDP_LOG_CRITICAL}, + {FDP::logging::OFF, FDP_LOG_OFF} +}; + + +void fdp_set_log_level(FDP_LOG_LEVEL log_level){ + FDP::logger::get_logger()->set_level(to_cpp_enum[log_level]); +} + + +FDP_LOG_LEVEL fdp_get_log_level(){ + return to_c_enum[FDP::logger::get_logger()->get_level()]; +} + + +int fdp_log(FDP_LOG_LEVEL log_level, const char* msg){ + switch(log_level){ + case FDP_LOG_TRACE: + FDP::logger::get_logger()->trace() << msg; + break; + case FDP_LOG_DEBUG: + FDP::logger::get_logger()->debug() << msg; + break; + case FDP_LOG_INFO: + FDP::logger::get_logger()->info() << msg; + break; + case FDP_LOG_WARN: + FDP::logger::get_logger()->warn() << msg; + break; + case FDP_LOG_ERROR: + FDP::logger::get_logger()->error() << msg; + break; + case FDP_LOG_CRITICAL: + FDP::logger::get_logger()->critical() << msg; + break; + default: + return 1; + } + return 0; +} From ba4cf72a123f83e5d5ca05207ca8e7fb83ad77a8 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 19 Apr 2023 09:49:08 +0100 Subject: [PATCH 59/74] Fix CI Warnings --- .github/workflows/docs.yaml | 6 +++--- .github/workflows/fdp_cpp_api.yaml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 120c06b..138d812 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.9" architecture: "x64" @@ -17,7 +17,7 @@ jobs: with: fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - name: build docs - uses: mattnotmitt/doxygen-action@v1.9.2 + uses: mattnotmitt/doxygen-action@v1.9.5 with: doxyfile-path: 'Doxyfile' - name: Deploy diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 1aaaae1..ccbe40a 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -9,8 +9,8 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.9" architecture: "x64" @@ -20,7 +20,7 @@ jobs: - name: Install local registry run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main - name: Checkout FAIRDataPipeline/FAIR-CLI - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: FAIRDataPipeline/FAIR-CLI path: FAIR-CLI @@ -47,7 +47,7 @@ jobs: run: | cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install - name: Fetch cache - uses: actions/cache@v2.1.5 + uses: actions/cache@v3 with: path: sonarCache key: ${{ runner.os }}-sonarCache-${{ github.sha }} @@ -98,7 +98,7 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install Dependencies run: | brew install cmake @@ -120,7 +120,7 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1 - name: Configure Library run: | @@ -139,7 +139,7 @@ jobs: image: ubuntu:18.04 options: --user root steps: - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: "3.9" architecture: "x64" From 58a96811612c8f345de7bcc820cc6856454f7232 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 11:27:22 +0100 Subject: [PATCH 60/74] Update workflows to use cSimpleModel test --- .github/workflows/test_with_simple_model.yaml | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index aba96d1..d03d6ff 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -21,13 +21,20 @@ jobs: sudo apt-get install -y gnuplot - name: Install local registry run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main - - name: Checkout Simple + - name: Checkout C++ Simple Model uses: actions/checkout@v2 with: repository: FAIRDataPipeline/cppSimpleModel - path: simpleModel + path: cppSimpleModel + - name: Checkout C Simple Model + uses: actions/checkout@v2 + with: + repository: PlasmaFAIR/cDataPipelineSimpleModel + path: cSimpleModel - name: Move simpleModel - run: mv simpleModel ../simpleModel + run: | + mv cppSimpleModel ../cppSimpleModel + mv cSimple_model ../cSimpleModel - name: Install Poetry uses: snok/install-poetry@v1 with: @@ -38,7 +45,7 @@ jobs: sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev - name: Build and run seirs example run: | - cd ../simpleModel + cd ../cppSimpleModel python3 -m venv venv source venv/bin/activate pip3 install fair-cli @@ -50,7 +57,31 @@ jobs: if: startsWith(github.ref, 'refs/tags/') != true - name: Build and run seirs example on tagged release run: | - cd ../simpleModel + cd ../cppSimpleModel + python3 -m venv venv + source venv/bin/activate + pip3 install fair-cli + cmake -Bbuild -DCPPDATAPIPELINEREF="tags/${GITHUB_REF/refs\/tags\//}" + cmake --build build + fair init --ci --local + fair pull --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml + if: startsWith(github.ref, 'refs/tags/') + - name: Build and run seirs example using C API + run: | + cd ../cSimpleModel + python3 -m venv venv + source venv/bin/activate + pip3 install fair-cli + cmake -Bbuild -DCPPDATAPIPELINEREF="heads/${GITHUB_REF/refs\/heads\//}" + cmake --build build + fair init --ci --local + fair pull --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml + if: startsWith(github.ref, 'refs/tags/') != true + - name: Build and run seirs example using C API on tagged release + run: | + cd ../cSimpleModel python3 -m venv venv source venv/bin/activate pip3 install fair-cli @@ -69,21 +100,34 @@ jobs: - uses: actions/checkout@v3 - name: Build run: | - cmake -B build -DCMAKE_INSTALL_PREFIX=install + cmake -B build -DCMAKE_INSTALL_PREFIX=../install cmake --build build --target install - name: Install graphviz run: | sudo apt update sudo apt-get install graphviz sudo apt-get install -y gnuplot - - name: Checkout SimpleModel + - name: Checkout C++ SimpleModel uses: actions/checkout@v3 with: repository: FAIRDataPipeline/cppSimpleModel - ref: prevent_fetchcontent_option - path: simpleModel - - name: Build Simple Model + path: cppSimpleModel + - name: Checkout C SimpleModel + uses: actions/checkout@v3 + with: + repository: PlasmaFAIR/cDataPipelineSimpleModel + path: cSimpleModel + - name: Move Simple Model dirs + run: | + mv cppSimpleModel ../cppSimpleModel + mv cSimpleModel ../cSimpleModel + - name: Build cpp Simple Model + run: | + cd ../cppSimpleModel + cmake -B build -DCMAKE_INSTALL_PREFIX=../install -DFDPAPI_NO_FETCHCONTENT=ON + cmake --build build + - name: Build C Simple Model run: | - cd simpleModel + cd ../cSimpleModel cmake -B build -DCMAKE_INSTALL_PREFIX=../install -DFDPAPI_NO_FETCHCONTENT=ON cmake --build build From c01bbd0687ac3896a8160ed2f2ccf5c36c54227f Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 11:32:43 +0100 Subject: [PATCH 61/74] Fix typo in simple model workflow --- .github/workflows/test_with_simple_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index d03d6ff..9226ffb 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -34,7 +34,7 @@ jobs: - name: Move simpleModel run: | mv cppSimpleModel ../cppSimpleModel - mv cSimple_model ../cSimpleModel + mv cSimpleModel ../cSimpleModel - name: Install Poetry uses: snok/install-poetry@v1 with: From 236f5f0c367f162a2a5c6b2745534ea268207173 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 11:52:03 +0100 Subject: [PATCH 62/74] Empty commit: Trigger CI to use updated cSimpleModel repo From 475fb653e98cbbf2164311a713006a22445d5a0c Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 12:00:30 +0100 Subject: [PATCH 63/74] Fix yaml file name for workflow running cSimpleModel --- .github/workflows/test_with_simple_model.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index 9226ffb..5475a0f 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -76,8 +76,8 @@ jobs: cmake -Bbuild -DCPPDATAPIPELINEREF="heads/${GITHUB_REF/refs\/heads\//}" cmake --build build fair init --ci --local - fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair pull --local data/config.yaml + fair run --local data/config.yaml if: startsWith(github.ref, 'refs/tags/') != true - name: Build and run seirs example using C API on tagged release run: | @@ -88,8 +88,8 @@ jobs: cmake -Bbuild -DCPPDATAPIPELINEREF="tags/${GITHUB_REF/refs\/tags\//}" cmake --build build fair init --ci --local - fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair pull --local data/config.yaml + fair run --local data/config.yaml if: startsWith(github.ref, 'refs/tags/') Build_simple_model_from_install: name: Build Ubuntu simple model from install From a9e257214a71e609cf4cc9e9049a430272c35e1b Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 16:15:44 +0100 Subject: [PATCH 64/74] Added tests for C API --- src/fdp_c_api.cxx | 1 + test/test_c_api.cxx | 74 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 test/test_c_api.cxx diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 74111a3..04f1a91 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -119,6 +119,7 @@ FDP_ERR_T fdp_finalise(){ return exception_to_err_code_void( [=](){ _datapipeline->finalise(); + _datapipeline = nullptr; } ); } diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx new file mode 100644 index 0000000..e1dcbc4 --- /dev/null +++ b/test/test_c_api.cxx @@ -0,0 +1,74 @@ +#ifndef TESTDIR +#define TESTDIR "" +#endif + +#include +#include +#include +#include + +#include "fdp/fdp.h" +#include "fdp/objects/metadata.hxx" // read_token + +#include +#include "gtest/gtest.h" + +namespace fs = ghc::filesystem; +namespace fdp = FairDataPipeline; + + +std::string home_dir(){ + std::string home; +#ifdef _WIN32 + home = getenv("HOMEDRIVE"); + home += getenv("HOMEPATH"); +#else + home = getenv("HOME"); +#endif + return home; +} + + +TEST(CTest, link_read_write){ + fdp_set_log_level(FDP_LOG_DEBUG); + + // Initialise + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = fdp::read_token( + fs::path(home_dir()) / ".fair" / "registry" / "token" + ); + ASSERT_EQ(fdp_init(config.c_str(), script.c_str(), token.c_str()), FDP_ERR_NONE); + char buf[512]; + + // Test link write + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_write("test/csv", buf), FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Write to new path + std::ofstream fstream(buf); + fstream << "Test"; + fstream.close(); + + // Finalise and re-initialise + ASSERT_EQ(fdp_finalise(), FDP_ERR_NONE); + config = fs::path(TESTDIR) / "data" / "read_csv.yaml"; + ASSERT_EQ(fdp_init(config.c_str(), script.c_str(), token.c_str()), FDP_ERR_NONE); + + // Test link read + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_read("test/csv", buf), FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Finalise again + EXPECT_EQ(fdp_finalise(), FDP_ERR_NONE); +} + + +TEST(CTest, log_levels){ + fdp_set_log_level(FDP_LOG_INFO); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_INFO); + fdp_set_log_level(FDP_LOG_DEBUG); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_DEBUG); +} From b28cda154ede50a5bacfa12f1114386c1ef62564 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 19 Apr 2023 16:43:21 +0100 Subject: [PATCH 65/74] Fix path-to-string conversion issue Fix bug on Windows where converting a path using c_str() results in a wchar_t* instead of the expected char* --- test/test_c_api.cxx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx index e1dcbc4..486b282 100644 --- a/test/test_c_api.cxx +++ b/test/test_c_api.cxx @@ -38,7 +38,10 @@ TEST(CTest, link_read_write){ std::string token = fdp::read_token( fs::path(home_dir()) / ".fair" / "registry" / "token" ); - ASSERT_EQ(fdp_init(config.c_str(), script.c_str(), token.c_str()), FDP_ERR_NONE); + ASSERT_EQ( + fdp_init(config.string().c_str(), script.string().c_str(), token.c_str()), + FDP_ERR_NONE + ); char buf[512]; // Test link write @@ -54,7 +57,10 @@ TEST(CTest, link_read_write){ // Finalise and re-initialise ASSERT_EQ(fdp_finalise(), FDP_ERR_NONE); config = fs::path(TESTDIR) / "data" / "read_csv.yaml"; - ASSERT_EQ(fdp_init(config.c_str(), script.c_str(), token.c_str()), FDP_ERR_NONE); + ASSERT_EQ( + fdp_init(config.string().c_str(), script.string().c_str(), token.c_str()), + FDP_ERR_NONE + ); // Test link read buf[0] = '\0'; // Ensure strlen of output buffer is 0 From 9a86e07b658ebf5d5a0a4fd1a61c4d5888d78d51 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 25 Apr 2023 16:43:09 +0100 Subject: [PATCH 66/74] C API passes struct FdpDataPipeline Rather than having the C API modify a global DataPipeline, now uses a struct 'FdpDataPipeline' which is passed to init, finalise, link_read and link_write. - Updated naming conventions - Fixed repeat value enum bug --- include/fdp/fdp.h | 80 +++++++++++++++++++++++++----- src/fdp_c_api.cxx | 117 +++++++++++++++++++++++++------------------- test/test_c_api.cxx | 23 ++++++--- 3 files changed, 152 insertions(+), 68 deletions(-) diff --git a/include/fdp/fdp.h b/include/fdp/fdp.h index f9b1142..fecdfd7 100644 --- a/include/fdp/fdp.h +++ b/include/fdp/fdp.h @@ -3,9 +3,23 @@ #ifdef __cplusplus + +#include "fdp.hxx" + extern "C" { + #endif +/** + * @brief Struct providing an interface to the pipeline. + * + * Defined in the implementation file, as it depends on C++ features. A pointer to this + * struct should be passed to functions in the C API. Set up by the function fdp_init, + * and finalised by fdp_finalise. Can also be generated from a C++ DataPipeline + * using to_c_struct. + */ +struct FdpDataPipeline; +typedef struct FdpDataPipeline FdpDataPipeline; /** * @brief Enumeration used to denote different error types. @@ -13,7 +27,7 @@ extern "C" { * The underlying C++ API will raise a number of different exception types. These map to * integer error codes for C compatibility. */ -enum FDP_ERR_T { +enum FdpError { FDP_ERR_NONE = 0, FDP_ERR_CONFIG_PARSE = 1, FDP_ERR_REST_API_QUERY = 2, @@ -21,10 +35,10 @@ enum FDP_ERR_T { FDP_ERR_VALIDATION = 4, FDP_ERR_SYNC = 5, FDP_ERR_WRITE = 6, - FDP_ERR_TOML = 6, - FDP_ERR_OTHER = 7 + FDP_ERR_TOML = 7, + FDP_ERR_OTHER = 8 }; -typedef enum FDP_ERR_T FDP_ERR_T; +typedef enum FdpError FdpError; /** @@ -33,6 +47,10 @@ typedef enum FDP_ERR_T FDP_ERR_T; * Should be called once before any calls to fdp_link_read or fdp_link_write. If called * more than once, returns FDP_ERR_OTHER. * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. The user + * should declare a pointer to a FdpDataPipeline, and pass its + * address to this function. This function will then initialise + * the pipeline. * @param config_file_path Path to the `config.yaml` file for this FDP run. Should * be at the location `${FDP_CONFIG_DIR}/config.yaml`. * @param script_file_path Path to the script which initiates this FDP run. Should @@ -42,7 +60,8 @@ typedef enum FDP_ERR_T FDP_ERR_T; * * @return Error code. */ -FDP_ERR_T fdp_init( +FdpError fdp_init( + FdpDataPipeline **data_pipeline, const char *config_file_path, const char *script_file_path, const char *token @@ -56,10 +75,13 @@ FDP_ERR_T fdp_init( * * Record all data products and meta data to the registry. Update the code run with all * appropriate meta data. + * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. This function + * finalises the FdpDataPipeline, and sets its pointer to NULL. * * @return Error code. */ -FDP_ERR_T fdp_finalise(); +FdpError fdp_finalise(FdpDataPipeline **data_pipeline); /** @@ -68,12 +90,17 @@ FDP_ERR_T fdp_finalise(); * * Must be called after fdp_init and before fdp_finalise. * + * @param data_pipeline Pointer to a FdpDataPipeline object. * @param data_product Path to the input file. * @param data_store_path Path to the assigned data store location. The user should * allocate sufficient memory beforehand. * @return Error code */ -FDP_ERR_T fdp_link_read(const char *data_product, char *data_store_path); +FdpError fdp_link_read( + FdpDataPipeline *data_pipeline, + const char *data_product, + char *data_store_path +); /** @@ -82,12 +109,17 @@ FDP_ERR_T fdp_link_read(const char *data_product, char *data_store_path); * * Must be called after fdp_init and before fdp_finalise. * + * @param data_pipeline Pointer to a FdpDataPipeline object. * @param data_product Path to the output file. * @param data_store_path Path to the assigned data store location. The user should * allocate sufficient memory beforehand. * @return Error code */ -FDP_ERR_T fdp_link_write(const char *data_product, char *data_store_path); +FdpError fdp_link_write( + FdpDataPipeline *data_pipeline, + const char *data_product, + char *data_store_path +); /** @@ -97,7 +129,7 @@ FDP_ERR_T fdp_link_write(const char *data_product, char *data_store_path); * level to `DEBUG` will include all log types except `TRACE`. These correspond to * the C++ logging levels `FairDataPipeline::logging::LOG_LEVEL`. */ -enum FDP_LOG_LEVEL { +enum FdpLogLevel { FDP_LOG_TRACE = 0, FDP_LOG_DEBUG = 1, FDP_LOG_INFO = 2, @@ -106,7 +138,7 @@ enum FDP_LOG_LEVEL { FDP_LOG_CRITICAL = 5, FDP_LOG_OFF = 6 }; -typedef enum FDP_LOG_LEVEL FDP_LOG_LEVEL; +typedef enum FdpLogLevel FdpLogLevel; /** @@ -114,7 +146,7 @@ typedef enum FDP_LOG_LEVEL FDP_LOG_LEVEL; * * @param log_level */ -void fdp_set_log_level(FDP_LOG_LEVEL log_level); +void fdp_set_log_level(FdpLogLevel log_level); /** @@ -122,7 +154,7 @@ void fdp_set_log_level(FDP_LOG_LEVEL log_level); * * @return Log level */ -FDP_LOG_LEVEL fdp_get_log_level(); +FdpLogLevel fdp_get_log_level(); /** @@ -135,11 +167,33 @@ FDP_LOG_LEVEL fdp_get_log_level(); * * @return Error code. 1 if logging unsuccessful, 0 otherwise. */ -int fdp_log(FDP_LOG_LEVEL log_level, const char* msg); +int fdp_log(FdpLogLevel log_level, const char *msg); #ifdef __cplusplus + } // close extern "C" + +namespace FairDataPipeline { + +/** + * @brief Convert data pipeline from the C API to one in the C++ API. + */ +DataPipeline::sptr from_c_struct(FdpDataPipeline *data_pipeline); + +/** + * @brief Convert data pipeline from the C++ API to one in the C API. + * + * If the pipeline is set up using the C++ method DataPipeline::construct, this may be + * used to generate a C-compatible struct. Note that this uses 'new' to allocate the + * returned pointer, so the user should 'delete` the pointer after use to avoid memory + * leaks. It is not recommended to mix usage of the C and C++ APIs for init and finalise + * functions. + */ +FdpDataPipeline* to_c_struct(DataPipeline::sptr data_pipeline); + +} // close namespace FairDataPipeline + #endif diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 04f1a91..646637e 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -11,12 +11,29 @@ namespace FDP = FairDataPipeline; + +// Define FdpDataPipeline struct and conversion routines + +struct FdpDataPipeline { + FDP::DataPipeline::sptr _pipeline; +}; + + +FDP::DataPipeline::sptr FDP::from_c_struct(FdpDataPipeline *data_pipeline){ + return data_pipeline->_pipeline; +} + + +FdpDataPipeline* FDP::to_c_struct(FDP::DataPipeline::sptr data_pipeline){ + return new FdpDataPipeline{data_pipeline}; +} + /** * @brief Utility method, calls exception-raising function and returns error codes * * Used to permit calls to C++ functions that may raise exceptions within a C * environment. If any exceptions are thrown, these are caught and converted to an error - * code using the enum type FDP_ERR_T. + * code using the enum type FdpError. * * Many functions in the C++ API operate on a shared pointer to a * FairDataPipeline::DataPipeline object. To call a function on this shared pointer @@ -31,7 +48,7 @@ namespace FDP = FairDataPipeline; * @see exception_to_err_code_void */ template -FDP_ERR_T exception_to_err_code(Function&& function, Return& ret, Args&&... args){ +FdpError exception_to_err_code(Function&& function, Return& ret, Args&&... args){ try{ ret = std::forward(function)(std::forward(args)...); return FDP_ERR_NONE; @@ -66,7 +83,7 @@ FDP_ERR_T exception_to_err_code(Function&& function, Return& ret, Args&&... args * @brief Companion to exception_to_err_code for non-returning functions. */ template -FDP_ERR_T exception_to_err_code_void(Function&& function, Args&&... args){ +FdpError exception_to_err_code_void(Function&& function, Args&&... args){ int dummy; return exception_to_err_code( [&function](Args&&... args) -> int { @@ -84,81 +101,83 @@ FDP_ERR_T exception_to_err_code_void(Function&& function, Args&&... args){ // ================= -/** - * @brief Global instance of FairDataPipeline::DataPipeline - * - * This should be initialised by fdp_init and deleted by fdp_finalise. The functions - * fdp_link_read and fdp_link_write make use of this, so will have undefined behaviour - * if called before init and after finalise. - */ -FDP::DataPipeline::sptr _datapipeline; - - -FDP_ERR_T fdp_init( +FdpError fdp_init( + FdpDataPipeline **data_pipeline, const char *config_file_path, const char *script_file_path, const char *token ){ std::string token_str = (token == nullptr ? "" : token); - - if(_datapipeline == nullptr){ - return exception_to_err_code( - FDP::DataPipeline::construct, - _datapipeline, - std::string(config_file_path), - std::string(script_file_path), - token_str - ); - } - return FDP_ERR_OTHER; + FDP::DataPipeline::sptr cpp_data_pipeline; + FdpError err = exception_to_err_code( + FDP::DataPipeline::construct, + cpp_data_pipeline, + std::string(config_file_path), + std::string(script_file_path), + token_str + ); + *data_pipeline = to_c_struct(cpp_data_pipeline); + return err; } -FDP_ERR_T fdp_finalise(){ - if(_datapipeline == nullptr) return FDP_ERR_OTHER; - return exception_to_err_code_void( - [=](){ - _datapipeline->finalise(); - _datapipeline = nullptr; - } +FdpError fdp_finalise(FdpDataPipeline **data_pipeline){ + if(*data_pipeline == nullptr || (*data_pipeline)->_pipeline == nullptr){ + return FDP_ERR_OTHER; + } + FdpError err = exception_to_err_code_void( + [](FDP::DataPipeline::sptr pipeline){pipeline->finalise();}, + (*data_pipeline)->_pipeline ); + delete *data_pipeline; + *data_pipeline = nullptr; + return err; } template -FDP_ERR_T _fdp_link(LinkFunction&& link_function, const char* path, char* output){ - if(_datapipeline == nullptr) return FDP_ERR_OTHER; +FdpError _fdp_link( + LinkFunction&& link_function, + FdpDataPipeline *data_pipeline, + const char *path, + char *output +){ + if(data_pipeline == nullptr || data_pipeline->_pipeline == nullptr){ + return FDP_ERR_OTHER; + } std::string input_path = path; std::string output_path; - FDP_ERR_T err = exception_to_err_code( + // Call either link_read or link_write on the pipeline, sets output_path + FdpError err = exception_to_err_code( std::forward(link_function), output_path, + data_pipeline->_pipeline, input_path ); - if(err){ - return err; - } + if(err) return err; strcpy(output, output_path.c_str()); return FDP_ERR_NONE; } -FDP_ERR_T fdp_link_read(const char* path, char* output){ +FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *path, char *output){ return _fdp_link( - [=](std::string& path) -> std::string { - return _datapipeline->link_read(path); + [](FDP::DataPipeline::sptr pipeline, std::string& path) -> std::string { + return pipeline->link_read(path); }, + data_pipeline, path, output ); } -FDP_ERR_T fdp_link_write(const char* path, char* output){ +FdpError fdp_link_write(FdpDataPipeline *data_pipeline, const char *path, char *output){ return _fdp_link( - [=](std::string& path) -> std::string { - return _datapipeline->link_write(path); + [](FDP::DataPipeline::sptr pipeline, std::string& path) -> std::string { + return pipeline->link_write(path); }, + data_pipeline, path, output ); @@ -172,7 +191,7 @@ FDP_ERR_T fdp_link_write(const char* path, char* output){ /** * @brief Map converting C API logging enums to the C++ API */ -std::map to_cpp_enum = { +std::map to_cpp_enum = { {FDP_LOG_TRACE, FDP::logging::TRACE}, {FDP_LOG_DEBUG, FDP::logging::DEBUG}, {FDP_LOG_INFO, FDP::logging::INFO}, @@ -186,7 +205,7 @@ std::map to_cpp_enum = { /** * @brief Map converting C++ API logging enums to the C API */ -std::map to_c_enum = { +std::map to_c_enum = { {FDP::logging::TRACE, FDP_LOG_TRACE}, {FDP::logging::DEBUG, FDP_LOG_DEBUG}, {FDP::logging::INFO, FDP_LOG_INFO}, @@ -197,17 +216,17 @@ std::map to_c_enum = { }; -void fdp_set_log_level(FDP_LOG_LEVEL log_level){ +void fdp_set_log_level(FdpLogLevel log_level){ FDP::logger::get_logger()->set_level(to_cpp_enum[log_level]); } -FDP_LOG_LEVEL fdp_get_log_level(){ +FdpLogLevel fdp_get_log_level(){ return to_c_enum[FDP::logger::get_logger()->get_level()]; } -int fdp_log(FDP_LOG_LEVEL log_level, const char* msg){ +int fdp_log(FdpLogLevel log_level, const char *msg){ switch(log_level){ case FDP_LOG_TRACE: FDP::logger::get_logger()->trace() << msg; diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx index 486b282..ca7120a 100644 --- a/test/test_c_api.cxx +++ b/test/test_c_api.cxx @@ -33,20 +33,26 @@ TEST(CTest, link_read_write){ fdp_set_log_level(FDP_LOG_DEBUG); // Initialise + FdpDataPipeline* pipeline; fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; fs::path script = fs::path(TESTDIR) / "test_script.sh"; std::string token = fdp::read_token( fs::path(home_dir()) / ".fair" / "registry" / "token" ); ASSERT_EQ( - fdp_init(config.string().c_str(), script.string().c_str(), token.c_str()), + fdp_init( + &pipeline, + config.string().c_str(), + script.string().c_str(), + token.c_str() + ), FDP_ERR_NONE ); char buf[512]; // Test link write buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_write("test/csv", buf), FDP_ERR_NONE); + EXPECT_EQ(fdp_link_write(pipeline, "test/csv", buf), FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); // Write to new path @@ -55,20 +61,25 @@ TEST(CTest, link_read_write){ fstream.close(); // Finalise and re-initialise - ASSERT_EQ(fdp_finalise(), FDP_ERR_NONE); + ASSERT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); config = fs::path(TESTDIR) / "data" / "read_csv.yaml"; ASSERT_EQ( - fdp_init(config.string().c_str(), script.string().c_str(), token.c_str()), + fdp_init( + &pipeline, + config.string().c_str(), + script.string().c_str(), + token.c_str() + ), FDP_ERR_NONE ); // Test link read buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_read("test/csv", buf), FDP_ERR_NONE); + EXPECT_EQ(fdp_link_read(pipeline, "test/csv", buf), FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); // Finalise again - EXPECT_EQ(fdp_finalise(), FDP_ERR_NONE); + EXPECT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); } From 03859a2bf5b969fbb57cb831493e4e6dd7e4a4ff Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Tue, 25 Apr 2023 17:30:58 +0100 Subject: [PATCH 67/74] Added tests for mixed C and C++ code Also added delete_c_struct function, which is needed as the struct FdpDataPipeline is an incomplete type. --- include/fdp/fdp.h | 7 ++++- src/fdp_c_api.cxx | 9 ++++-- test/test_c_api.cxx | 75 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/include/fdp/fdp.h b/include/fdp/fdp.h index fecdfd7..e971c53 100644 --- a/include/fdp/fdp.h +++ b/include/fdp/fdp.h @@ -186,12 +186,17 @@ DataPipeline::sptr from_c_struct(FdpDataPipeline *data_pipeline); * * If the pipeline is set up using the C++ method DataPipeline::construct, this may be * used to generate a C-compatible struct. Note that this uses 'new' to allocate the - * returned pointer, so the user should 'delete` the pointer after use to avoid memory + * returned pointer, so the user should 'delete_c_struct` after use to avoid memory * leaks. It is not recommended to mix usage of the C and C++ APIs for init and finalise * functions. */ FdpDataPipeline* to_c_struct(DataPipeline::sptr data_pipeline); +/** + * @brief Function to clean up FdpDataPipeline created by to_c_struct + */ +void delete_c_struct(FdpDataPipeline *data_pipeline); + } // close namespace FairDataPipeline #endif diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 646637e..adbf3e4 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -28,6 +28,11 @@ FdpDataPipeline* FDP::to_c_struct(FDP::DataPipeline::sptr data_pipeline){ return new FdpDataPipeline{data_pipeline}; } + +void FDP::delete_c_struct(FdpDataPipeline *data_pipeline){ + delete data_pipeline; +} + /** * @brief Utility method, calls exception-raising function and returns error codes * @@ -116,7 +121,7 @@ FdpError fdp_init( std::string(script_file_path), token_str ); - *data_pipeline = to_c_struct(cpp_data_pipeline); + *data_pipeline = FDP::to_c_struct(cpp_data_pipeline); return err; } @@ -129,7 +134,7 @@ FdpError fdp_finalise(FdpDataPipeline **data_pipeline){ [](FDP::DataPipeline::sptr pipeline){pipeline->finalise();}, (*data_pipeline)->_pipeline ); - delete *data_pipeline; + FDP::delete_c_struct(*data_pipeline); *data_pipeline = nullptr; return err; } diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx index ca7120a..0a42dcd 100644 --- a/test/test_c_api.cxx +++ b/test/test_c_api.cxx @@ -31,6 +31,7 @@ std::string home_dir(){ TEST(CTest, link_read_write){ fdp_set_log_level(FDP_LOG_DEBUG); + char buf[512]; // Initialise FdpDataPipeline* pipeline; @@ -48,7 +49,6 @@ TEST(CTest, link_read_write){ ), FDP_ERR_NONE ); - char buf[512]; // Test link write buf[0] = '\0'; // Ensure strlen of output buffer is 0 @@ -82,6 +82,79 @@ TEST(CTest, link_read_write){ EXPECT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); } +TEST(CTest, cpp_to_c){ + fdp_set_log_level(FDP_LOG_DEBUG); + char buf[512]; + + // Initialise using C++ + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = fdp::read_token( + fs::path(home_dir()) / ".fair" / "registry" / "token" + ); + auto cpp_pipeline = fdp::DataPipeline::construct( + config.string(), script.string(), token + ); + + // Switch to C API, use temporary FdpDataPipeline + FdpDataPipeline *c_pipeline = fdp::to_c_struct(cpp_pipeline); + + // Test link write + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_write(c_pipeline, "test/csv", buf), FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Write to new path + std::ofstream fstream(buf); + fstream << "Test"; + fstream.close(); + + // Finish working in C, delete FdpDataPipeline + fdp::delete_c_struct(c_pipeline); + + // Finalise in C++ + cpp_pipeline->finalise(); +} + +TEST(CTest, c_to_cpp){ + fdp_set_log_level(FDP_LOG_DEBUG); + + // Initialise using C + FdpDataPipeline* c_pipeline; + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = fdp::read_token( + fs::path(home_dir()) / ".fair" / "registry" / "token" + ); + ASSERT_EQ( + fdp_init( + &c_pipeline, + config.string().c_str(), + script.string().c_str(), + token.c_str() + ), + FDP_ERR_NONE + ); + + // Switch to C++ API + auto cpp_pipeline = fdp::from_c_struct(c_pipeline); + + // Test link write + std::string data_product = "test/csv"; + fs::path currentLink = fs::path(cpp_pipeline->link_write(data_product)); + EXPECT_GT(currentLink.string().size(), 1); + + // Write to new path + std::ofstream fstream(currentLink); + fstream << "Test"; + fstream.close(); + + // Finish working in C++ + cpp_pipeline = nullptr; + + // Finalise in C + EXPECT_EQ(fdp_finalise(&c_pipeline), FDP_ERR_NONE); +} TEST(CTest, log_levels){ fdp_set_log_level(FDP_LOG_INFO); From 9c286da6ceb35fa8e30e53f554b56db28864e9b5 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 28 Jun 2023 09:00:15 +0100 Subject: [PATCH 68/74] remove gcc 4.8 --- .github/workflows/fdp_cpp_api.yaml | 42 +----------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index ccbe40a..5033926 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -131,44 +131,4 @@ jobs: run: | cmake --build build --config=Release --target install cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON - cmake --build build_from_install --config=Release - Build_GCC_4_8: - name: Build GCC 4.8 - runs-on: ubuntu-latest - container: - image: ubuntu:18.04 - options: --user root - steps: - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: "x64" - - name: Setup GCC 4.8 - run: | - apt update - apt install build-essential libssl-dev git -y - git clone https://github.com/Kitware/CMake/ - cd CMake - ./bootstrap && make && make install - cmake --version - apt install g++-4.8 gcc-4.8 -y - rm /usr/bin/gcc - rm /usr/bin/g++ - ln -s /usr/bin/gcc-4.8 /usr/bin/gcc - ln -s /usr/bin/g++-4.8 /usr/bin/g++ - gcc --version - g++ --version - cmake --version - - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - apt install -y lcov curl libcurl4-openssl-dev gcovr - - name: Configure Library - run: | - cd $GITHUB_WORKSPACE - ls - cmake -Bbuild - - name: Build Library - run: | - cd $GITHUB_WORKSPACE - cmake --build build + cmake --build build_from_install --config=Release \ No newline at end of file From 5ee1a99542a841b4b45b95894945059561b4eb4e Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 28 Jun 2023 09:34:43 +0100 Subject: [PATCH 69/74] switch to std::regex --- CMakeLists.txt | 1 - cmake_modules/fdpapiConfig.cmake.in | 1 - external/re2.cmake | 7 ------- include/fdp/objects/config.hxx | 2 +- include/fdp/objects/metadata.hxx | 2 +- include/fdp/registry/api.hxx | 2 +- src/CMakeLists.txt | 1 - src/objects/config.cxx | 3 +-- src/objects/metadata.cxx | 8 ++------ src/registry/api.cxx | 31 ++++++++++------------------- test/CMakeLists.txt | 1 - 11 files changed, 17 insertions(+), 42 deletions(-) delete mode 100644 external/re2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e1aa5f3..03692ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,6 @@ include(external/curl.cmake) include(external/yaml_cpp.cmake) include(external/toml11.cmake) include(external/ghc.cmake) -include(external/re2.cmake) include(external/digestpp.cmake) # Define and install library diff --git a/cmake_modules/fdpapiConfig.cmake.in b/cmake_modules/fdpapiConfig.cmake.in index b95a475..e084592 100644 --- a/cmake_modules/fdpapiConfig.cmake.in +++ b/cmake_modules/fdpapiConfig.cmake.in @@ -8,7 +8,6 @@ find_package(digestpp) find_package(ghc_filesystem) find_package(jsoncpp) find_package(yaml-cpp) -find_package(re2) include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") diff --git a/external/re2.cmake b/external/re2.cmake deleted file mode 100644 index cf7f4c0..0000000 --- a/external/re2.cmake +++ /dev/null @@ -1,7 +0,0 @@ -set(RE2_BUILD_TESTING OFF CACHE INTERNAL "") -fdpapi_add_external( - "RE2" - REPO "https://github.com/google/re2.git" - TAG "2022-12-01" - PKG_NAME "re2" -) diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index f70e68e..1710233 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index d07f84d..1494ce7 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include "digestpp/digestpp.hpp" diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index 25d8f7d..f061c22 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ab35efa..c66a8a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -54,7 +54,6 @@ target_link_libraries(fdpapi PRIVATE toml11::toml11) target_link_libraries(fdpapi PRIVATE digestpp::digestpp) target_link_libraries(fdpapi PRIVATE CURL::libcurl) target_link_libraries(fdpapi PRIVATE yaml-cpp) -target_link_libraries(fdpapi PRIVATE re2::re2) target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) if(BUILD_SHARED_LIBS) target_link_libraries(fdpapi PRIVATE jsoncpp_lib) diff --git a/src/objects/config.cxx b/src/objects/config.cxx index 29533d5..08b1e0c 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -326,8 +326,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { Json::Value j_code_repo_root = api_->post("storage_root", repo_storage_root_value_, token_); this->code_repo_storage_root_ = ApiObject::from_json( j_code_repo_root ); - std::string repo_storage_path_ = meta_data_()["remote_repo"].as(); - RE2::GlobalReplace(&repo_storage_path_, repo_storage_root_value_["root"].asString(), ""); + std::string repo_storage_path_ = std::regex_replace(meta_data_()["remote_repo"].as(), std::regex(repo_storage_root_value_["root"].asString()), ""); Json::Value repo_storage_location_value_; repo_storage_location_value_["hash"] = meta_data_()["latest_commit"].as(); diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 6c23bdd..6860aa2 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -64,15 +64,11 @@ std::string current_time_stamp(bool file_name) { } std::string remove_local_from_root(const std::string &root){ - std::string result = root; - RE2::GlobalReplace(&result, "file:\\/\\/", ""); - return result; + return std::regex_replace(root, std::regex(std::string("file:\\/\\/")), ""); } std::string remove_backslash_from_path(const std::string &path){ - std::string result = path; - RE2::GlobalReplace(&result, "\\\\", "/"); - return result; + return std::regex_replace(path, std::regex(std::string("\\\\")), "/"); } bool file_exists( const std::string &Filename ) diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 4970014..b3bc638 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -95,8 +95,7 @@ CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, Json::Value API::get_request(const ghc::filesystem::path &addr_path, long expected_response, std::string token) { - std::string addr_path_ = addr_path.string(); - RE2::GlobalReplace(&addr_path_, "\\\\", "/"); + std::string addr_path_ = std::regex_replace(addr_path.string(), std::regex(std::string("\\\\")), "/"); return get_request(addr_path_, expected_response); } @@ -168,8 +167,6 @@ std::string API::json_to_query_string(Json::Value &json_value) { std::string rtn = "?"; // Need to remove the api address from any values using regex std::string regex_string = "(" + url_root_ + ")([A-Za-z_]+)\\/([0-9]+)\\/"; - std::string match1, match2; - int match3; // Check the json value is not empty if (json_value.size() > 0) { // Iterate through the json keys @@ -182,20 +179,18 @@ std::string API::json_to_query_string(Json::Value &json_value) { i++) { // add the key and value to the return string after removing the api // address with regex - std::string str = json_value.get(key, "")[i].asString(); - if(RE2::FullMatch(str, regex_string, &match1, &match2, &match3)){ - str = std::to_string(match3); - } - rtn += key + "=" + str + "&"; + rtn += key + "=" + + std::regex_replace(json_value.get(key, "")[i].asString(), + std::regex(regex_string), "$3") + + "&"; } } else { // if it's not an array add the key and value to the return string after // removing the api address with regex - std::string str = json_value.get(key, "").asString(); - if(RE2::FullMatch(str, regex_string, &match1, &match2, &match3)){ - str = std::to_string(match3); - } - rtn += key + "=" + str + "&"; + rtn += key + "=" + + std::regex_replace(json_value.get(key, "").asString(), + std::regex(regex_string), "$3") + + "&"; } } } @@ -205,9 +200,7 @@ std::string API::json_to_query_string(Json::Value &json_value) { std::string API::escape_space(std::string &str) { // Using regex replace space with html character (%20) - std::string result = str; - RE2::GlobalReplace(&result, " ", "%20"); - return result; + return std::string(std::regex_replace(str, std::regex(" "), "%20")); } Json::Value API::post(std::string addr_path, Json::Value &post_data, @@ -229,9 +222,7 @@ Json::Value API::post_file_type(Json::Value &post_data, const std::string &token logger::get_logger()->error() << "Error: Post Data does not contain a file extension"; throw rest_apiquery_error("Failed to post file_type"); } - std::string extension = post_data["extension"].asString(); - RE2::GlobalReplace(&extension, ".", ""); - post_data["extension"] = extension; + post_data["extension"] = regex_replace(post_data["name"].asString(), std::regex("."), ""); Json::Value _file_type_query; _file_type_query["extension"] = post_data["extension"]; Json::Value _file_type_exists = get_by_json_query("file_type", _file_type_query); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 32705e2..4bb99ab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -64,7 +64,6 @@ target_link_libraries(fdpapi-tests PRIVATE gtest gtest_main) target_link_libraries(fdpapi-tests PRIVATE toml11::toml11) target_link_libraries(fdpapi-tests PRIVATE digestpp::digestpp) -target_link_libraries(fdpapi-tests PRIVATE re2::re2) target_link_libraries(fdpapi-tests PRIVATE CURL::libcurl) target_link_libraries(fdpapi-tests PRIVATE yaml-cpp) if(BUILD_SHARED_LIBS) From 2aa26e200fdfcb4bce2761f6cedc859e0d4fd816 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 28 Jun 2023 10:58:22 +0100 Subject: [PATCH 70/74] Fix to potential std::forward error Unsure how std::forward interacts with lambda functions and capture-by-reference. C++20 template lambdas could handle this more easily. --- src/fdp_c_api.cxx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index adbf3e4..4e062d8 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -91,12 +91,11 @@ template FdpError exception_to_err_code_void(Function&& function, Args&&... args){ int dummy; return exception_to_err_code( - [&function](Args&&... args) -> int { + [&function, &args...]() -> int { std::forward(function)(std::forward(args)...); return 0; }, - dummy, - std::forward(args)... + dummy ); } From 9a07ae648c54961db1b759bebf34f8fdb3fcd14f Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 28 Jun 2023 12:16:30 +0100 Subject: [PATCH 71/74] clang-format, improved string security in C API Used clang-format on C API bits Better tests for valid strings/buffers in the C API Better error messages, and exits earlier when encountering errors. --- include/fdp/fdp.h | 191 +++++++++++---------- src/fdp_c_api.cxx | 402 +++++++++++++++++++++++++------------------- test/test_c_api.cxx | 95 +++++------ 3 files changed, 362 insertions(+), 326 deletions(-) diff --git a/include/fdp/fdp.h b/include/fdp/fdp.h index e971c53..d46bf11 100644 --- a/include/fdp/fdp.h +++ b/include/fdp/fdp.h @@ -1,7 +1,6 @@ #ifndef __FDP_C_API__ #define __FDP_C_API__ - #ifdef __cplusplus #include "fdp.hxx" @@ -13,10 +12,10 @@ extern "C" { /** * @brief Struct providing an interface to the pipeline. * - * Defined in the implementation file, as it depends on C++ features. A pointer to this - * struct should be passed to functions in the C API. Set up by the function fdp_init, - * and finalised by fdp_finalise. Can also be generated from a C++ DataPipeline - * using to_c_struct. + * Defined in the implementation file, as it depends on C++ features. A pointer + * to this struct should be passed to functions in the C API. Set up by the + * function fdp_init, and finalised by fdp_finalise. Can also be generated from + * a C++ DataPipeline using to_c_struct. */ struct FdpDataPipeline; typedef struct FdpDataPipeline FdpDataPipeline; @@ -24,123 +23,121 @@ typedef struct FdpDataPipeline FdpDataPipeline; /** * @brief Enumeration used to denote different error types. * - * The underlying C++ API will raise a number of different exception types. These map to - * integer error codes for C compatibility. + * The underlying C++ API will raise a number of different exception types. + * These map to integer error codes for C compatibility. */ enum FdpError { - FDP_ERR_NONE = 0, - FDP_ERR_CONFIG_PARSE = 1, - FDP_ERR_REST_API_QUERY = 2, - FDP_ERR_JSON_PARSE = 3, - FDP_ERR_VALIDATION = 4, - FDP_ERR_SYNC = 5, - FDP_ERR_WRITE = 6, - FDP_ERR_TOML = 7, - FDP_ERR_OTHER = 8 + FDP_ERR_NONE = 0, + FDP_ERR_CONFIG_PARSE = 1, + FDP_ERR_REST_API_QUERY = 2, + FDP_ERR_JSON_PARSE = 3, + FDP_ERR_VALIDATION = 4, + FDP_ERR_SYNC = 5, + FDP_ERR_WRITE = 6, + FDP_ERR_TOML = 7, + FDP_ERR_OTHER = 8 }; typedef enum FdpError FdpError; - /** * @brief Initialise the pipeline. * - * Should be called once before any calls to fdp_link_read or fdp_link_write. If called - * more than once, returns FDP_ERR_OTHER. - * - * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. The user - * should declare a pointer to a FdpDataPipeline, and pass its - * address to this function. This function will then initialise - * the pipeline. - * @param config_file_path Path to the `config.yaml` file for this FDP run. Should - * be at the location `${FDP_CONFIG_DIR}/config.yaml`. - * @param script_file_path Path to the script which initiates this FDP run. Should - * be at the location `${FDP_CONFIG_DIR}/script.sh` (or - * `${FDP_CONFIG_DIR}/script.bat` on Windows). - * @param token Token used to connect to FDP registry. May be set to `NULL`. + * Should be called once before any calls to fdp_link_read or fdp_link_write. If + * called more than once, returns FDP_ERR_OTHER. + * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. The + * user should declare a pointer to a FdpDataPipeline, and pass its address to + * this function. This function will then initialise the pipeline. + * + * @param config_file_path Path to the `config.yaml` file for this FDP run. + * Should be at the location `${FDP_CONFIG_DIR}/config.yaml`. + * + * @param script_file_path Path to the script which initiates this FDP run. + * Should be at the location `${FDP_CONFIG_DIR}/script.sh` on Linux/Mac or + * `${FDP_CONFIG_DIR}/script.bat` on Windows. + * + * @param token Token used to connect to FDP registry. May be set to `NULL`. * * @return Error code. */ -FdpError fdp_init( - FdpDataPipeline **data_pipeline, - const char *config_file_path, - const char *script_file_path, - const char *token -); - +FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, + const char *script_file_path, const char *token); /** * @brief Finalise the pipeline. * * Must be called after a call to fdp_init. - * - * Record all data products and meta data to the registry. Update the code run with all - * appropriate meta data. * - * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. This function - * finalises the FdpDataPipeline, and sets its pointer to NULL. - * + * Record all data products and meta data to the registry. Update the code run + * with all appropriate meta data. + * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. This + * function finalises the FdpDataPipeline, and sets its pointer to NULL. + * * @return Error code. */ FdpError fdp_finalise(FdpDataPipeline **data_pipeline); - /** - * @brief Set a path to a given data product while recording it's meta data for the - * code run. + * @brief Set a path to a given data product while recording its meta data for + * the code run. * * Must be called after fdp_init and before fdp_finalise. - * - * @param data_pipeline Pointer to a FdpDataPipeline object. - * @param data_product Path to the input file. - * @param data_store_path Path to the assigned data store location. The user should - * allocate sufficient memory beforehand. + * + * @param data_pipeline Pointer to a FdpDataPipeline object. + * + * @param data_product Path to the input file. + * + * @param data_store_path Path to the assigned data store location. The user + * should allocate sufficient memory beforehand. + * + * @param data_store_path_len Size of the buffer used for the data store + * path. + * * @return Error code */ -FdpError fdp_link_read( - FdpDataPipeline *data_pipeline, - const char *data_product, - char *data_store_path -); - +FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *data_product, + char *data_store_path, size_t data_store_path_len); /** - * @brief Set a path to a given data product while recording it's meta data for the - * code run. + * @brief Set a path to a given data product while recording its meta data for + * the code run. * * Must be called after fdp_init and before fdp_finalise. - * - * @param data_pipeline Pointer to a FdpDataPipeline object. - * @param data_product Path to the output file. - * @param data_store_path Path to the assigned data store location. The user should - * allocate sufficient memory beforehand. + * + * @param data_pipeline Pointer to a FdpDataPipeline object. + * + * @param data_product Path to the output file. + * + * @param data_store_path Path to the assigned data store location. The user + * should allocate sufficient memory beforehand. + * + * @param data_store_path_len Size of the buffer used for the data store + * path. + * * @return Error code */ -FdpError fdp_link_write( - FdpDataPipeline *data_pipeline, - const char *data_product, - char *data_store_path -); - +FdpError fdp_link_write(FdpDataPipeline *data_pipeline, + const char *data_product, char *data_store_path, size_t data_store_path_len); /** * @brief Enumeration used to denote the different levels of logging. * - * Each level of logging includes all log levels greater than it, so setting the log - * level to `DEBUG` will include all log types except `TRACE`. These correspond to - * the C++ logging levels `FairDataPipeline::logging::LOG_LEVEL`. + * Each level of logging includes all log levels greater than it, so setting the + * log level to `DEBUG` will include all log types except `TRACE`. These + * correspond to the C++ logging levels `FairDataPipeline::logging::LOG_LEVEL`. */ enum FdpLogLevel { - FDP_LOG_TRACE = 0, - FDP_LOG_DEBUG = 1, - FDP_LOG_INFO = 2, - FDP_LOG_WARN = 3, - FDP_LOG_ERROR = 4, - FDP_LOG_CRITICAL = 5, - FDP_LOG_OFF = 6 + FDP_LOG_TRACE = 0, + FDP_LOG_DEBUG = 1, + FDP_LOG_INFO = 2, + FDP_LOG_WARN = 3, + FDP_LOG_ERROR = 4, + FDP_LOG_CRITICAL = 5, + FDP_LOG_OFF = 6 }; typedef enum FdpLogLevel FdpLogLevel; - /** * @brief Set the log level. Must call `fdp_init` first. * @@ -148,7 +145,6 @@ typedef enum FdpLogLevel FdpLogLevel; */ void fdp_set_log_level(FdpLogLevel log_level); - /** * @brief Get the current log level. Must call fdp_init first. * @@ -156,20 +152,22 @@ void fdp_set_log_level(FdpLogLevel log_level); */ FdpLogLevel fdp_get_log_level(); - /** - * @brief Write a message to the log. This will be passed to the C++ logger, - * `FairDataPipeline::logger::get_logger()->level() << msg`, where `level` - * is one of `trace`, `debug`, `info`, `warn`, `error`, or `critical`. + * @brief Write a message to the log. + * + * This will be passed to the C++ logger, + * `FairDataPipeline::logger::get_logger()->level() << msg`, where + * `level` is one of `trace`, `debug`, `info`, `warn`, `error`, or `critical`. * - * @param log_level The type of log message to write, e.g. FDP_LOG_INFO, FDP_LOG_ERROR. - * @param msg The message to be written to log. + * @param log_level The type of log message to write, e.g. FDP_LOG_INFO, + * FDP_LOG_ERROR. + * + * @param msg The message to be written to log. * * @return Error code. 1 if logging unsuccessful, 0 otherwise. */ int fdp_log(FdpLogLevel log_level, const char *msg); - #ifdef __cplusplus } // close extern "C" @@ -184,22 +182,21 @@ DataPipeline::sptr from_c_struct(FdpDataPipeline *data_pipeline); /** * @brief Convert data pipeline from the C++ API to one in the C API. * - * If the pipeline is set up using the C++ method DataPipeline::construct, this may be - * used to generate a C-compatible struct. Note that this uses 'new' to allocate the - * returned pointer, so the user should 'delete_c_struct` after use to avoid memory - * leaks. It is not recommended to mix usage of the C and C++ APIs for init and finalise - * functions. + * If the pipeline is set up using the C++ method DataPipeline::construct, this + * may be used to generate a C-compatible struct. Note that this uses 'new' to + * allocate the returned pointer, so the user should 'delete_c_struct` after use + * to avoid memory leaks. It is not recommended to mix usage of the C and C++ + * APIs for init and finalise functions. */ -FdpDataPipeline* to_c_struct(DataPipeline::sptr data_pipeline); +FdpDataPipeline *to_c_struct(DataPipeline::sptr data_pipeline); /** * @brief Function to clean up FdpDataPipeline created by to_c_struct */ void delete_c_struct(FdpDataPipeline *data_pipeline); -} // close namespace FairDataPipeline +} // namespace FairDataPipeline #endif - #endif // __FDP_C_API__ diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 4e062d8..15e67d7 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -1,197 +1,261 @@ +#include +#include #include -#include +#include #include +#include #include +#include "fdp/exceptions.hxx" #include "fdp/fdp.h" #include "fdp/fdp.hxx" -#include "fdp/exceptions.hxx" #include "fdp/utilities/logging.hxx" - namespace FDP = FairDataPipeline; - // Define FdpDataPipeline struct and conversion routines struct FdpDataPipeline { - FDP::DataPipeline::sptr _pipeline; + FDP::DataPipeline::sptr _pipeline; }; - -FDP::DataPipeline::sptr FDP::from_c_struct(FdpDataPipeline *data_pipeline){ - return data_pipeline->_pipeline; +FDP::DataPipeline::sptr FDP::from_c_struct(FdpDataPipeline *data_pipeline) { + return data_pipeline->_pipeline; } - -FdpDataPipeline* FDP::to_c_struct(FDP::DataPipeline::sptr data_pipeline){ - return new FdpDataPipeline{data_pipeline}; +FdpDataPipeline *FDP::to_c_struct(FDP::DataPipeline::sptr data_pipeline) { + return new FdpDataPipeline{data_pipeline}; } - -void FDP::delete_c_struct(FdpDataPipeline *data_pipeline){ - delete data_pipeline; +void FDP::delete_c_struct(FdpDataPipeline *data_pipeline) { + delete data_pipeline; } -/** - * @brief Utility method, calls exception-raising function and returns error codes +/** + * @brief Utility method, calls exception-raising function and returns error + * codes * * Used to permit calls to C++ functions that may raise exceptions within a C - * environment. If any exceptions are thrown, these are caught and converted to an error - * code using the enum type FdpError. + * environment. If any exceptions are thrown, these are caught and converted to + * an error code using the enum type FdpError. * * Many functions in the C++ API operate on a shared pointer to a - * FairDataPipeline::DataPipeline object. To call a function on this shared pointer - * using exception_to_err_code, use a lambda that captures the shared pointer. + * FairDataPipeline::DataPipeline object. To call a function on this shared + * pointer using exception_to_err_code, use a lambda that captures the shared + * pointer. * * @param function The C++ function to call. Must not have a void return type. - * @param ret Parameter in which to store the result of the function call. - * @param args Args to pass to the function. - * + * + * @param ret Parameter in which to store the result of the function call. + * + * @param args Args to pass to the function. + * * @return Error code - * + * * @see exception_to_err_code_void */ -template -FdpError exception_to_err_code(Function&& function, Return& ret, Args&&... args){ - try{ - ret = std::forward(function)(std::forward(args)...); - return FDP_ERR_NONE; - } - catch(const FDP::config_parsing_error&){ - return FDP_ERR_CONFIG_PARSE; - } - catch(const FDP::rest_apiquery_error&){ - return FDP_ERR_REST_API_QUERY; - } - catch(const FDP::json_parse_error&){ - return FDP_ERR_JSON_PARSE; - } - catch(const FDP::validation_error&){ - return FDP_ERR_VALIDATION; - } - catch(const FDP::sync_error&){ - return FDP_ERR_SYNC; - } - catch(const FDP::write_error&){ - return FDP_ERR_WRITE; - } - catch(const FDP::toml_error&){ - return FDP_ERR_TOML; - } - catch(...){ - return FDP_ERR_OTHER; - } +template +FdpError exception_to_err_code(Function &&function, Return &ret, + Args &&... args) { + try { + ret = std::forward(function)(std::forward(args)...); + return FDP_ERR_NONE; + } catch (const FDP::config_parsing_error &) { + return FDP_ERR_CONFIG_PARSE; + } catch (const FDP::rest_apiquery_error &) { + return FDP_ERR_REST_API_QUERY; + } catch (const FDP::json_parse_error &) { + return FDP_ERR_JSON_PARSE; + } catch (const FDP::validation_error &) { + return FDP_ERR_VALIDATION; + } catch (const FDP::sync_error &) { + return FDP_ERR_SYNC; + } catch (const FDP::write_error &) { + return FDP_ERR_WRITE; + } catch (const FDP::toml_error &) { + return FDP_ERR_TOML; + } catch (...) { + return FDP_ERR_OTHER; + } } -/** +/** * @brief Companion to exception_to_err_code for non-returning functions. */ -template -FdpError exception_to_err_code_void(Function&& function, Args&&... args){ - int dummy; - return exception_to_err_code( - [&function, &args...]() -> int { - std::forward(function)(std::forward(args)...); - return 0; - }, - dummy - ); +template +FdpError exception_to_err_code_void(Function &&function, Args &&... args) { + int dummy; + return exception_to_err_code( + [&function, &args...]() -> int { + std::forward(function)(std::forward(args)...); + return 0; + }, + dummy); } +/** + * @brief Get error name from error code + */ +std::string error_name(FdpError err) { + std::string result; + switch (err) { + FDP_ERR_NONE: + result = "None"; + break; + FDP_ERR_CONFIG_PARSE: + result = "Config Parse"; + break; + FDP_ERR_REST_API_QUERY: + result = "REST API Query"; + break; + FDP_ERR_JSON_PARSE: + result = "JSON Parse"; + break; + FDP_ERR_VALIDATION: + result = "Validation"; + break; + FDP_ERR_SYNC: + result = "Sync"; + break; + FDP_ERR_WRITE: + result = "Write"; + break; + FDP_ERR_TOML: + result = "TOML"; + break; + default: + result = "Other"; + } + return result; +} // ================= // init and finalise // ================= - -FdpError fdp_init( - FdpDataPipeline **data_pipeline, - const char *config_file_path, - const char *script_file_path, - const char *token -){ - std::string token_str = (token == nullptr ? "" : token); - FDP::DataPipeline::sptr cpp_data_pipeline; - FdpError err = exception_to_err_code( - FDP::DataPipeline::construct, - cpp_data_pipeline, - std::string(config_file_path), - std::string(script_file_path), - token_str - ); - *data_pipeline = FDP::to_c_struct(cpp_data_pipeline); +FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, + const char *script_file_path, const char *token) { + std::string token_str = (token == nullptr ? "" : token); + FDP::DataPipeline::sptr cpp_data_pipeline; + FdpError err = exception_to_err_code( + FDP::DataPipeline::construct, cpp_data_pipeline, + std::string(config_file_path), std::string(script_file_path), token_str); + if (err) { + std::cerr << "ERROR: Error of type '" << error_name(err) + << "' in call to fdp_init" << std::endl; + *data_pipeline = nullptr; return err; + } + *data_pipeline = FDP::to_c_struct(cpp_data_pipeline); + return err; } - -FdpError fdp_finalise(FdpDataPipeline **data_pipeline){ - if(*data_pipeline == nullptr || (*data_pipeline)->_pipeline == nullptr){ - return FDP_ERR_OTHER; - } - FdpError err = exception_to_err_code_void( - [](FDP::DataPipeline::sptr pipeline){pipeline->finalise();}, - (*data_pipeline)->_pipeline - ); - FDP::delete_c_struct(*data_pipeline); - *data_pipeline = nullptr; +FdpError fdp_finalise(FdpDataPipeline **data_pipeline) { + if (*data_pipeline == nullptr || (*data_pipeline)->_pipeline == nullptr) { + std::cerr << "ERROR: Pipeline not initialiased in call to fdp_finalise" + << std::endl; + return FDP_ERR_OTHER; + } + FdpError err = exception_to_err_code_void( + [](FDP::DataPipeline::sptr pipeline) { pipeline->finalise(); }, + (*data_pipeline)->_pipeline); + if (err) { + std::cerr << "ERROR: Error of type '" << error_name(err) + << "' in call to fdp_finalise" << std::endl; return err; + } + FDP::delete_c_struct(*data_pipeline); + *data_pipeline = nullptr; + return err; } - -template -FdpError _fdp_link( - LinkFunction&& link_function, - FdpDataPipeline *data_pipeline, - const char *path, - char *output -){ - if(data_pipeline == nullptr || data_pipeline->_pipeline == nullptr){ - return FDP_ERR_OTHER; +template +FdpError _fdp_link(LinkFunction &&link_function, + const std::string &link_function_name, + FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + // Ensure pipeline is initialised + if (data_pipeline == nullptr || data_pipeline->_pipeline == nullptr) { + std::cerr << "ERROR: Data pipeline not initialised in call to " + << link_function_name << std::endl; + return FDP_ERR_OTHER; + } + // Ensure input and output paths are valid + if (path == nullptr) { + std::cerr << "ERROR: Input path is NULL in call to " << link_function_name + << std::endl; + return FDP_ERR_OTHER; + } + if (output == nullptr) { + std::cerr << "ERROR: Output path is NULL in call to " << link_function_name + << std::endl; + return FDP_ERR_OTHER; + } + // Check input is null terminated, max 4096 chars, including terminator + // TODO Should we check MAX_PATH/PATH_MAX here? + bool path_null_terminated = false; + std::size_t path_len = 0; + for (std::size_t ii = 0; ii < 4096; ++ii) { + if (path[ii] == '\0') { + path_null_terminated = true; + path_len = ii; + break; } - std::string input_path = path; - std::string output_path; - // Call either link_read or link_write on the pipeline, sets output_path - FdpError err = exception_to_err_code( - std::forward(link_function), - output_path, - data_pipeline->_pipeline, - input_path - ); - if(err) return err; - strcpy(output, output_path.c_str()); - return FDP_ERR_NONE; + } + if (!path_null_terminated) { + std::cerr << "ERROR: Input path is not null-terminated or is longer than " + "4095 chars in call to " + << link_function_name << std::endl; + return FDP_ERR_OTHER; + } + // Check input and output don't overlap + auto x1 = reinterpret_cast(path); + auto x2 = x1 + path_len; + auto y1 = reinterpret_cast(output); + auto y2 = y1 + output_len; + if (std::max(x1, y1) <= std::min(x2, y2)) { + std::cerr << "ERROR: Input and output paths overlap in call to " + << link_function_name << std::endl; + return FDP_ERR_OTHER; + } + // Use C++ strings over C strings to interface with the C++ pipeline + std::string input_path = path; + std::string output_path; + // Call either link_read or link_write on the pipeline, sets output_path + FdpError err = + exception_to_err_code(std::forward(link_function), + output_path, data_pipeline->_pipeline, input_path); + if (err) { + std::cerr << "ERROR: Error of type '" << error_name(err) << "' in call to " + << link_function_name << std::endl; + return err; + } + strncat(output, output_path.c_str(), output_len); + return FDP_ERR_NONE; } - -FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *path, char *output){ - return _fdp_link( - [](FDP::DataPipeline::sptr pipeline, std::string& path) -> std::string { - return pipeline->link_read(path); - }, - data_pipeline, - path, - output - ); +FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + return _fdp_link( + [](FDP::DataPipeline::sptr pipeline, std::string &path) -> std::string { + return pipeline->link_read(path); + }, + "fdp_link_read", data_pipeline, path, output, output_len); } - -FdpError fdp_link_write(FdpDataPipeline *data_pipeline, const char *path, char *output){ - return _fdp_link( - [](FDP::DataPipeline::sptr pipeline, std::string& path) -> std::string { - return pipeline->link_write(path); - }, - data_pipeline, - path, - output - ); +FdpError fdp_link_write(FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + return _fdp_link( + [](FDP::DataPipeline::sptr pipeline, std::string &path) -> std::string { + return pipeline->link_write(path); + }, + "fdp_link_write", data_pipeline, path, output, output_len); } // ======= // logging // ======= - /** * @brief Map converting C API logging enums to the C++ API */ @@ -202,9 +266,7 @@ std::map to_cpp_enum = { {FDP_LOG_WARN, FDP::logging::WARN}, {FDP_LOG_ERROR, FDP::logging::ERROR}, {FDP_LOG_CRITICAL, FDP::logging::CRITICAL}, - {FDP_LOG_OFF, FDP::logging::OFF} -}; - + {FDP_LOG_OFF, FDP::logging::OFF}}; /** * @brief Map converting C++ API logging enums to the C API @@ -216,42 +278,38 @@ std::map to_c_enum = { {FDP::logging::WARN, FDP_LOG_WARN}, {FDP::logging::ERROR, FDP_LOG_ERROR}, {FDP::logging::CRITICAL, FDP_LOG_CRITICAL}, - {FDP::logging::OFF, FDP_LOG_OFF} -}; - + {FDP::logging::OFF, FDP_LOG_OFF}}; -void fdp_set_log_level(FdpLogLevel log_level){ - FDP::logger::get_logger()->set_level(to_cpp_enum[log_level]); +void fdp_set_log_level(FdpLogLevel log_level) { + FDP::logger::get_logger()->set_level(to_cpp_enum[log_level]); } - -FdpLogLevel fdp_get_log_level(){ - return to_c_enum[FDP::logger::get_logger()->get_level()]; +FdpLogLevel fdp_get_log_level() { + return to_c_enum[FDP::logger::get_logger()->get_level()]; } - -int fdp_log(FdpLogLevel log_level, const char *msg){ - switch(log_level){ - case FDP_LOG_TRACE: - FDP::logger::get_logger()->trace() << msg; - break; - case FDP_LOG_DEBUG: - FDP::logger::get_logger()->debug() << msg; - break; - case FDP_LOG_INFO: - FDP::logger::get_logger()->info() << msg; - break; - case FDP_LOG_WARN: - FDP::logger::get_logger()->warn() << msg; - break; - case FDP_LOG_ERROR: - FDP::logger::get_logger()->error() << msg; - break; - case FDP_LOG_CRITICAL: - FDP::logger::get_logger()->critical() << msg; - break; - default: - return 1; - } - return 0; +int fdp_log(FdpLogLevel log_level, const char *msg) { + switch (log_level) { + case FDP_LOG_TRACE: + FDP::logger::get_logger()->trace() << msg; + break; + case FDP_LOG_DEBUG: + FDP::logger::get_logger()->debug() << msg; + break; + case FDP_LOG_INFO: + FDP::logger::get_logger()->info() << msg; + break; + case FDP_LOG_WARN: + FDP::logger::get_logger()->warn() << msg; + break; + case FDP_LOG_ERROR: + FDP::logger::get_logger()->error() << msg; + break; + case FDP_LOG_CRITICAL: + FDP::logger::get_logger()->critical() << msg; + break; + default: + return 1; + } + return 0; } diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx index 0a42dcd..a15679c 100644 --- a/test/test_c_api.cxx +++ b/test/test_c_api.cxx @@ -3,21 +3,20 @@ #endif #include -#include #include +#include #include #include "fdp/fdp.h" #include "fdp/objects/metadata.hxx" // read_token -#include #include "gtest/gtest.h" +#include namespace fs = ghc::filesystem; namespace fdp = FairDataPipeline; - -std::string home_dir(){ +std::string home_dir() { std::string home; #ifdef _WIN32 home = getenv("HOMEDRIVE"); @@ -28,31 +27,26 @@ std::string home_dir(){ return home; } +#define BUFFER_SIZE 512 -TEST(CTest, link_read_write){ +TEST(CTest, link_read_write) { fdp_set_log_level(FDP_LOG_DEBUG); - char buf[512]; + char buf[BUFFER_SIZE]; // Initialise - FdpDataPipeline* pipeline; + FdpDataPipeline *pipeline; fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; fs::path script = fs::path(TESTDIR) / "test_script.sh"; - std::string token = fdp::read_token( - fs::path(home_dir()) / ".fair" / "registry" / "token" - ); - ASSERT_EQ( - fdp_init( - &pipeline, - config.string().c_str(), - script.string().c_str(), - token.c_str() - ), - FDP_ERR_NONE - ); + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + ASSERT_EQ(fdp_init(&pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); // Test link write buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_write(pipeline, "test/csv", buf), FDP_ERR_NONE); + EXPECT_EQ(fdp_link_write(pipeline, "test/csv", buf, BUFFER_SIZE), + FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); // Write to new path @@ -63,45 +57,39 @@ TEST(CTest, link_read_write){ // Finalise and re-initialise ASSERT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); config = fs::path(TESTDIR) / "data" / "read_csv.yaml"; - ASSERT_EQ( - fdp_init( - &pipeline, - config.string().c_str(), - script.string().c_str(), - token.c_str() - ), - FDP_ERR_NONE - ); + ASSERT_EQ(fdp_init(&pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); // Test link read buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_read(pipeline, "test/csv", buf), FDP_ERR_NONE); + EXPECT_EQ(fdp_link_read(pipeline, "test/csv", buf, BUFFER_SIZE), + FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); // Finalise again EXPECT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); } -TEST(CTest, cpp_to_c){ +TEST(CTest, cpp_to_c) { fdp_set_log_level(FDP_LOG_DEBUG); char buf[512]; // Initialise using C++ fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; fs::path script = fs::path(TESTDIR) / "test_script.sh"; - std::string token = fdp::read_token( - fs::path(home_dir()) / ".fair" / "registry" / "token" - ); - auto cpp_pipeline = fdp::DataPipeline::construct( - config.string(), script.string(), token - ); + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + auto cpp_pipeline = + fdp::DataPipeline::construct(config.string(), script.string(), token); // Switch to C API, use temporary FdpDataPipeline FdpDataPipeline *c_pipeline = fdp::to_c_struct(cpp_pipeline); // Test link write buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_write(c_pipeline, "test/csv", buf), FDP_ERR_NONE); + EXPECT_EQ(fdp_link_write(c_pipeline, "test/csv", buf, BUFFER_SIZE), + FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); // Write to new path @@ -116,25 +104,18 @@ TEST(CTest, cpp_to_c){ cpp_pipeline->finalise(); } -TEST(CTest, c_to_cpp){ +TEST(CTest, c_to_cpp) { fdp_set_log_level(FDP_LOG_DEBUG); // Initialise using C - FdpDataPipeline* c_pipeline; + FdpDataPipeline *c_pipeline; fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; fs::path script = fs::path(TESTDIR) / "test_script.sh"; - std::string token = fdp::read_token( - fs::path(home_dir()) / ".fair" / "registry" / "token" - ); - ASSERT_EQ( - fdp_init( - &c_pipeline, - config.string().c_str(), - script.string().c_str(), - token.c_str() - ), - FDP_ERR_NONE - ); + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + ASSERT_EQ(fdp_init(&c_pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); // Switch to C++ API auto cpp_pipeline = fdp::from_c_struct(c_pipeline); @@ -156,9 +137,9 @@ TEST(CTest, c_to_cpp){ EXPECT_EQ(fdp_finalise(&c_pipeline), FDP_ERR_NONE); } -TEST(CTest, log_levels){ - fdp_set_log_level(FDP_LOG_INFO); - EXPECT_EQ(fdp_get_log_level(), FDP_LOG_INFO); - fdp_set_log_level(FDP_LOG_DEBUG); - EXPECT_EQ(fdp_get_log_level(), FDP_LOG_DEBUG); +TEST(CTest, log_levels) { + fdp_set_log_level(FDP_LOG_INFO); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_INFO); + fdp_set_log_level(FDP_LOG_DEBUG); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_DEBUG); } From 4421b47b71ff7d14a02bb34b1125c579210cd4cb Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 28 Jun 2023 12:30:21 +0100 Subject: [PATCH 72/74] Replace std::cerr IO with logger error calls in C API --- src/fdp_c_api.cxx | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 15e67d7..910881e 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -141,8 +140,7 @@ FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, FDP::DataPipeline::construct, cpp_data_pipeline, std::string(config_file_path), std::string(script_file_path), token_str); if (err) { - std::cerr << "ERROR: Error of type '" << error_name(err) - << "' in call to fdp_init" << std::endl; + // Trust that the C++ API logged the error before throwing *data_pipeline = nullptr; return err; } @@ -152,16 +150,15 @@ FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, FdpError fdp_finalise(FdpDataPipeline **data_pipeline) { if (*data_pipeline == nullptr || (*data_pipeline)->_pipeline == nullptr) { - std::cerr << "ERROR: Pipeline not initialiased in call to fdp_finalise" - << std::endl; + FDP::logger::get_logger()->error() + << "Pipeline not initialiased in call to fdp_finalise"; return FDP_ERR_OTHER; } FdpError err = exception_to_err_code_void( [](FDP::DataPipeline::sptr pipeline) { pipeline->finalise(); }, (*data_pipeline)->_pipeline); if (err) { - std::cerr << "ERROR: Error of type '" << error_name(err) - << "' in call to fdp_finalise" << std::endl; + // Trust that the C++ API logged the error before throwing return err; } FDP::delete_c_struct(*data_pipeline); @@ -176,19 +173,19 @@ FdpError _fdp_link(LinkFunction &&link_function, char *output, size_t output_len) { // Ensure pipeline is initialised if (data_pipeline == nullptr || data_pipeline->_pipeline == nullptr) { - std::cerr << "ERROR: Data pipeline not initialised in call to " - << link_function_name << std::endl; + FDP::logger::get_logger()->error() + << " Data pipeline not initialised in call to " << link_function_name; return FDP_ERR_OTHER; } // Ensure input and output paths are valid if (path == nullptr) { - std::cerr << "ERROR: Input path is NULL in call to " << link_function_name - << std::endl; + FDP::logger::get_logger()->error() + << "Input path is NULL in call to " << link_function_name; return FDP_ERR_OTHER; } if (output == nullptr) { - std::cerr << "ERROR: Output path is NULL in call to " << link_function_name - << std::endl; + FDP::logger::get_logger()->error() + << "Output path is NULL in call to " << link_function_name; return FDP_ERR_OTHER; } // Check input is null terminated, max 4096 chars, including terminator @@ -203,9 +200,10 @@ FdpError _fdp_link(LinkFunction &&link_function, } } if (!path_null_terminated) { - std::cerr << "ERROR: Input path is not null-terminated or is longer than " - "4095 chars in call to " - << link_function_name << std::endl; + FDP::logger::get_logger()->error() + << "Input path is not null-terminated or is longer than 4095 chars in " + "call to " + << link_function_name; return FDP_ERR_OTHER; } // Check input and output don't overlap @@ -214,8 +212,8 @@ FdpError _fdp_link(LinkFunction &&link_function, auto y1 = reinterpret_cast(output); auto y2 = y1 + output_len; if (std::max(x1, y1) <= std::min(x2, y2)) { - std::cerr << "ERROR: Input and output paths overlap in call to " - << link_function_name << std::endl; + FDP::logger::get_logger()->error() + << "Input and output paths overlap in call to " << link_function_name; return FDP_ERR_OTHER; } // Use C++ strings over C strings to interface with the C++ pipeline @@ -226,8 +224,7 @@ FdpError _fdp_link(LinkFunction &&link_function, exception_to_err_code(std::forward(link_function), output_path, data_pipeline->_pipeline, input_path); if (err) { - std::cerr << "ERROR: Error of type '" << error_name(err) << "' in call to " - << link_function_name << std::endl; + // Trust that the C++ API logged the error before throwing return err; } strncat(output, output_path.c_str(), output_len); From ec11afa8fd8df42b04bb6d6c16c7c580ae414f64 Mon Sep 17 00:00:00 2001 From: Liam Pattinson Date: Wed, 28 Jun 2023 13:53:00 +0100 Subject: [PATCH 73/74] Further fixes to string handling in C API --- src/fdp_c_api.cxx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx index 910881e..078610e 100644 --- a/src/fdp_c_api.cxx +++ b/src/fdp_c_api.cxx @@ -188,6 +188,12 @@ FdpError _fdp_link(LinkFunction &&link_function, << "Output path is NULL in call to " << link_function_name; return FDP_ERR_OTHER; } + // Check output len is valid + if (output_len == 0) { + FDP::logger::get_logger()->error() + << "output_len is zero in call to " << link_function_name; + return FDP_ERR_OTHER; + } // Check input is null terminated, max 4096 chars, including terminator // TODO Should we check MAX_PATH/PATH_MAX here? bool path_null_terminated = false; @@ -227,7 +233,14 @@ FdpError _fdp_link(LinkFunction &&link_function, // Trust that the C++ API logged the error before throwing return err; } - strncat(output, output_path.c_str(), output_len); + // Don't copy if output_path won't fit in output buffer + // Use >= instead of > to account for null terminator + if (output_path.size() >= output_len) { + FDP::logger::get_logger()->error() + << "Output path won't fit in buffer in call to " << link_function_name; + return FDP_ERR_OTHER; + } + strncpy(output, output_path.c_str(), output_len); return FDP_ERR_NONE; } From 54e404c5d60275554cb8ac730b7e46d8575d7194 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 28 Jun 2023 14:39:04 +0100 Subject: [PATCH 74/74] update test yaml's --- test/data/read_csv.yaml | 3 +++ test/data/write_csv.yaml | 5 +++++ test/test_c_api.cxx | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/data/read_csv.yaml b/test/data/read_csv.yaml index 09eb983..cd11b56 100644 --- a/test/data/read_csv.yaml +++ b/test/data/read_csv.yaml @@ -13,5 +13,8 @@ run_metadata: read: - data_product: test/csv + use: + version: 0.0.1 +- data_product: test/csv/c use: version: 0.0.1 \ No newline at end of file diff --git a/test/data/write_csv.yaml b/test/data/write_csv.yaml index 1b42fea..b941968 100644 --- a/test/data/write_csv.yaml +++ b/test/data/write_csv.yaml @@ -14,6 +14,11 @@ run_metadata: write: - data_product: test/csv + description: test csv file with simple data + file_type: csv + use: + version: 0.0.1 +- data_product: test/csv/c description: test csv file with simple data file_type: csv use: diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx index a15679c..d1ac2c7 100644 --- a/test/test_c_api.cxx +++ b/test/test_c_api.cxx @@ -45,7 +45,7 @@ TEST(CTest, link_read_write) { // Test link write buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_write(pipeline, "test/csv", buf, BUFFER_SIZE), + EXPECT_EQ(fdp_link_write(pipeline, "test/csv/c", buf, BUFFER_SIZE), FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1); @@ -63,7 +63,7 @@ TEST(CTest, link_read_write) { // Test link read buf[0] = '\0'; // Ensure strlen of output buffer is 0 - EXPECT_EQ(fdp_link_read(pipeline, "test/csv", buf, BUFFER_SIZE), + EXPECT_EQ(fdp_link_read(pipeline, "test/csv/c", buf, BUFFER_SIZE), FDP_ERR_NONE); EXPECT_GT(strlen(buf), 1);