diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 120c06b..138d812 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.9" architecture: "x64" @@ -17,7 +17,7 @@ jobs: with: fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - name: build docs - uses: mattnotmitt/doxygen-action@v1.9.2 + uses: mattnotmitt/doxygen-action@v1.9.5 with: doxyfile-path: 'Doxyfile' - name: Deploy diff --git a/.github/workflows/fdp_cpp_api.yaml b/.github/workflows/fdp_cpp_api.yaml index 108ae1d..5033926 100644 --- a/.github/workflows/fdp_cpp_api.yaml +++ b/.github/workflows/fdp_cpp_api.yaml @@ -1,6 +1,6 @@ name: FDP C++ API -on: [push] +on: [push, workflow_dispatch] jobs: Build_Ubuntu: @@ -9,8 +9,8 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.9" architecture: "x64" @@ -20,7 +20,7 @@ jobs: - name: Install local registry run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main - name: Checkout FAIRDataPipeline/FAIR-CLI - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: FAIRDataPipeline/FAIR-CLI path: FAIR-CLI @@ -42,12 +42,12 @@ jobs: - name: Install Dependencies run: | sudo apt update - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install - name: Fetch cache - uses: actions/cache@v2.1.5 + uses: actions/cache@v3 with: path: sonarCache key: ${{ runner.os }}-sonarCache-${{ github.sha }} @@ -79,57 +79,56 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - name: Build from Install + run: | + cmake --build build --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DFDPAPI_CODE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON + cmake --build build_from_install + build-wrapper-linux-x86-64 --out-dir bw-outputs cmake --build build --target coverage + if [ $? -eq 0 ]; then + echo "Unit tests completed successfully" + exit 0 + else + echo "Unit tests failed" + exit 1 + fi Build_MacOS: name: Build MacOS runs-on: macOS-latest strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install Dependencies run: | brew install cmake brew install curl - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install - name: Build Library run: | cmake --build build + - name: Build from Install + run: | + cmake --build build --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON + cmake --build build_from_install Build_Windows: name: Build Windows runs-on: windows-latest strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1 - name: Configure Library run: | - cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON + cmake -Bbuild -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install - name: Compile FDP-Cpp-API run: cmake --build build --config=Release - Build_GCC_4_8: - name: Build GCC 4.8 - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: "3.9" - architecture: "x64" - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1 - with: - version: 4.8 - platform: x64 - - name: Install Dependencies + - name: Build from Install run: | - sudo apt update - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev gcovr - - name: Configure Library - run: | - cmake -Bbuild - - name: Build Library - run: | - cmake --build build + cmake --build build --config=Release --target install + cmake -B build_from_install -DFDPAPI_BUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=install -DFDPAPI_NEVER_FETCH=ON + cmake --build build_from_install --config=Release \ No newline at end of file diff --git a/.github/workflows/test_with_simple_model.yaml b/.github/workflows/test_with_simple_model.yaml index bbfda34..f7fb1ee 100644 --- a/.github/workflows/test_with_simple_model.yaml +++ b/.github/workflows/test_with_simple_model.yaml @@ -1,6 +1,6 @@ name: C++ Test Simple Model -on: [push] +on: [push, workflow_dispatch] jobs: Test_Simple_Model: @@ -11,23 +11,30 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - with: + with: python-version: "3.9" architecture: "x64" - name: Install graphviz run: | sudo apt update - sudo apt-get install graphviz + sudo apt-get install graphviz sudo apt-get install -y gnuplot - name: Install local registry run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main - - name: Checkout Simple + - name: Checkout C++ Simple Model uses: actions/checkout@v2 with: repository: FAIRDataPipeline/cppSimpleModel - path: simpleModel + path: cppSimpleModel + - name: Checkout C Simple Model + uses: actions/checkout@v2 + with: + repository: PlasmaFAIR/cDataPipelineSimpleModel + path: cSimpleModel - name: Move simpleModel - run: mv simpleModel ../simpleModel + run: | + mv cppSimpleModel ../cppSimpleModel + mv cSimpleModel ../cSimpleModel - name: Install Poetry uses: snok/install-poetry@v1 with: @@ -35,10 +42,10 @@ jobs: virtualenvs-in-project: true - name: Install Dependencies run: | - sudo apt install -y lcov libjsoncpp-dev curl libcurl4-openssl-dev libyaml-cpp-dev + sudo apt install -y lcov curl libcurl4-openssl-dev libyaml-cpp-dev - name: Build and run seirs example run: | - cd ../simpleModel + cd ../cppSimpleModel python3 -m venv venv source venv/bin/activate pip3 install fair-cli @@ -46,11 +53,11 @@ jobs: cmake --build build fair init --ci --local fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml if: startsWith(github.ref, 'refs/tags/') != true - name: Build and run seirs example on tagged release run: | - cd ../simpleModel + cd ../cppSimpleModel python3 -m venv venv source venv/bin/activate pip3 install fair-cli @@ -58,5 +65,69 @@ jobs: cmake --build build fair init --ci --local fair pull --local data/seirs_config.yaml - fair run --local data/seirs_config.yaml + fair run --local data/seirs_config.yaml if: startsWith(github.ref, 'refs/tags/') + - name: Build and run seirs example using C API + run: | + cd ../cSimpleModel + python3 -m venv venv + source venv/bin/activate + pip3 install fair-cli + cmake -Bbuild -DCPPDATAPIPELINEREF="heads/${GITHUB_REF/refs\/heads\//}" + cmake --build build + fair init --ci --local + fair pull --local data/config.yaml + fair run --local data/config.yaml + if: startsWith(github.ref, 'refs/tags/') != true + - name: Build and run seirs example using C API on tagged release + run: | + cd ../cSimpleModel + python3 -m venv venv + source venv/bin/activate + pip3 install fair-cli + cmake -Bbuild -DCPPDATAPIPELINEREF="tags/${GITHUB_REF/refs\/tags\//}" + cmake --build build + fair init --ci --local + fair pull --local data/config.yaml + fair run --local data/config.yaml + if: startsWith(github.ref, 'refs/tags/') + Build_simple_model_from_install: + name: Build Ubuntu simple model from install + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + - name: Build + run: | + cmake -B build -DCMAKE_INSTALL_PREFIX=../install + cmake --build build --target install + - name: Install graphviz + run: | + sudo apt update + sudo apt-get install graphviz + sudo apt-get install -y gnuplot + - name: Checkout C++ SimpleModel + uses: actions/checkout@v3 + with: + repository: FAIRDataPipeline/cppSimpleModel + path: cppSimpleModel + - name: Checkout C SimpleModel + uses: actions/checkout@v3 + with: + repository: PlasmaFAIR/cDataPipelineSimpleModel + path: cSimpleModel + - name: Move Simple Model dirs + run: | + mv cppSimpleModel ../cppSimpleModel + mv cSimpleModel ../cSimpleModel + - name: Build cpp Simple Model + run: | + cd ../cppSimpleModel + cmake -B build -DCMAKE_INSTALL_PREFIX=../install -DFDPAPI_NO_FETCHCONTENT=ON + cmake --build build + - name: Build C Simple Model + run: | + cd ../cSimpleModel + cmake -B build -DCMAKE_INSTALL_PREFIX=../install -DFDPAPI_NO_FETCHCONTENT=ON + cmake --build build diff --git a/.gitignore b/.gitignore index 23c7e14..339c4c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build/ +install/ .DS_STORE .vscode .bash_profile @@ -8,3 +9,4 @@ data_store/ venv/ test/data/temp/ Testing/ +CMakeFiles/ diff --git a/CMakeLists.txt b/CMakeLists.txt index ecc6edc..03692ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,11 @@ option(BUILD_SHARED_LIBS "Build Static Libraries" OFF) option(FDPAPI_BUILD_TESTS "Build unit tests" OFF) option(FDPAPI_CODE_COVERAGE "Run GCov and LCov code coverage tools" OFF) option(FDPAPI_WITH_INSTALL "Allow project to be installable" ON) +option(FDPAPI_ALWAYS_FETCH "Don't use pre-installed dependencies, use FetchContent instead" OFF) +option(FDPAPI_NEVER_FETCH "Only use pre-installed dependencies, don't use FetchContent" OFF) +if(FDPAPI_ALWAYS_FETCH AND FDPAPI_NEVER_FETCH) + message(FATAL_ERROR "FDPAPI_ALWAYS_FETCH and FDPAPI_NEVER_FETCH are mutually exclusive") +endif() # Disable Building in Debug as HD5 get_property(isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) @@ -35,6 +40,7 @@ else() set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release") endif() + # Set Output Directories to avoid issues on multi release compilers set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -49,13 +55,15 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() +# Include macro for adding external packages +include(cmake_modules/fdpapi_add_external.cmake) + # Include external library files include(external/jsoncpp.cmake) include(external/curl.cmake) include(external/yaml_cpp.cmake) include(external/toml11.cmake) include(external/ghc.cmake) -include(external/boost_regex.cmake) include(external/digestpp.cmake) # Define and install library diff --git a/cmake_modules/fdpapiConfig.cmake.in b/cmake_modules/fdpapiConfig.cmake.in index 945f7d4..e084592 100644 --- a/cmake_modules/fdpapiConfig.cmake.in +++ b/cmake_modules/fdpapiConfig.cmake.in @@ -8,7 +8,6 @@ find_package(digestpp) find_package(ghc_filesystem) find_package(jsoncpp) find_package(yaml-cpp) -find_package(Boost COMPONENTS regex QUIET) include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") diff --git a/cmake_modules/fdpapi_add_external.cmake b/cmake_modules/fdpapi_add_external.cmake new file mode 100644 index 0000000..d83932f --- /dev/null +++ b/cmake_modules/fdpapi_add_external.cmake @@ -0,0 +1,101 @@ +# Macro for finding packages, or installing them from online repos using FetchContent +# +# Required args +# ------------- +# pkg +# Name of the package. This should be a simplified version of the full package name, +# ideally in all-caps. It does not need to match the package name that is passed +# to find_package() +# +# Optional args +# ------------- +# PKG_NAME pkg_name +# The 'true' package name, which will be passed to find_package(). If not set, +# defaults to ${pkg} +# URL url +# The URL from which to download the package if find_package() is not successful. +# Usually set to a Github .zip archive. +# REPO repo +# The URL of a Git repository to pull from. Ignored if URL is set. It is recommended +# to always set TAG alongside this, or else the package will be pulled from the +# latest commits to main/master, which may be unstable. +# TAG tag +# The Git tag to use from REPO. Can be a commit hash, release tag, etc. +# +# Environment vars +# ---------------- +# FDPAPI_ALWAYS_FETCH_${pkg} +# Set to ON to skip find_package() and always use FetchContent for this package. +# FDPAPI_NEVER_FETCH_${pkg} +# Set to ON to skip FetchContent for this package. If find_package() is not +# successful, the build will fail. + +macro(fdpapi_add_external pkg) + message(STATUS "[${pkg}]") + cmake_parse_arguments(FDPAPI_${pkg} "" "PKG_NAME;URL;REPO;TAG" "" ${ARGN}) + + # Set default PKG_NAME. This is passed to find_package + if(NOT DEFINED FDPAPI_${pkg}_PKG_NAME) + set(FDPAPI_${pkg}_PKG_NAME ${pkg}) + endif() + + # Set options to allow the user to control how they wish to use this package + option( + FDPAPI_NEVER_FETCH_${pkg} + "Only use pre-installed ${pkg}, don't use FetchContent." + OFF + ) + option( + FDPAPI_ALWAYS_FETCH_${pkg} + "Don't use pre-installed ${pkg}, always use FetchContent." + OFF + ) + + if(FDPAPI_ALWAYS_FETCH_${pkg} AND FDPAPI_NEVER_FETCH_${pkg}) + message( + FATAL_ERROR + "FDPAPI_ALWAYS_FETCH_${pkg} and FDPAPI_NEVER_FETCH_${pkg} are mutually exclusive" + ) + endif() + + # Try to find package, unless the user has requested not to + if(NOT (FDPAPI_ALWAYS_FETCH_${pkg} OR FDPAPI_ALWAYS_FETCH)) + find_package(${FDPAPI_${pkg}_PKG_NAME} QUIET) + endif() + + # If found, write a message and continue. Otherwise, try FetchContent. + if(${FDPAPI_${pkg}_PKG_NAME}_FOUND AND NOT (FDPAPI_ALWAYS_FETCH_${pkg} OR FDPAPI_ALWAYS_FETCH)) + message(STATUS "\t${pkg} found.") + elseif(NOT (FDPAPI_NEVER_FETCH_${pkg} OR FDPAPI_NEVER_FETCH)) + message(STATUS "\t${pkg} not found.") + + # Prefer to use URL. Otherwise, use REPO, and prefer to use TAG in that case. + if(DEFINED FDPAPI_${pkg}_URL) + message(STATUS "\tInstalling from ${FDPAPI_${pkg}_URL}.") + FetchContent_Declare( + ${pkg} + URL "${FDPAPI_${pkg}_URL}" + ) + elseif(DEFINED FDPAPI_${pkg}_REPO) + message(STATUS "\tInstalling from ${FDPAPI_${pkg}_REPO}.") + if(DEFINED FDPAPI_${pkg}_TAG) + message(STATUS "\tGit tag: ${FDPAPI_${pkg}_TAG}.") + FetchContent_Declare( + ${pkg} + GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" + GIT_TAG "${FDPAPI_${pkg}_TAG}" + ) + else() + FetchContent_Declare( + ${pkg} + GIT_REPOSITORY "${FDPAPI_${pkg}_REPO}" + ) + endif() + else() + message(FATAL_ERROR "\t${pkg} could not be installed.") + endif() + FetchContent_MakeAvailable(${pkg}) + else() + message(FATAL_ERROR "\t${pkg} not found.") + endif() +endmacro() diff --git a/external/boost_regex.cmake b/external/boost_regex.cmake deleted file mode 100644 index df5ab80..0000000 --- a/external/boost_regex.cmake +++ /dev/null @@ -1,19 +0,0 @@ -message(STATUS "[Boost Regex]") - -set(BOOST_ENABLE_CMAKE ON CACHE INTERNAL "") - -find_package(Boost COMPONENTS regex) -if(NOT Boost_FOUND) - set(BRX_URL "https://github.com/boostorg/regex/archive/refs/tags/boost-1.79.0.zip") - - message(STATUS "\tBoost Regex Will be installed.") - message(STATUS "\tURL: ${BRX_URL}") - set(BOOST_REGEX_STANDALONE ON CACHE INTERNAL "Get just regex from Boost libraries") - - FetchContent_Declare( - BOOSTREGEX - URL ${BRX_URL} - ) - FetchContent_MakeAvailable(BOOSTREGEX) - -endif() diff --git a/external/curl.cmake b/external/curl.cmake index 2faf1dd..1e0cea0 100644 --- a/external/curl.cmake +++ b/external/curl.cmake @@ -1,23 +1,4 @@ -# Try and find CURL -FIND_PACKAGE( CURL QUIET ) -MESSAGE( STATUS "[Curl]" ) - -# If CURL is not Found, Install it -IF(NOT CURL_FOUND) - - SET( CURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip" ) - - MESSAGE( STATUS "\tCURL Will be installed." ) - MESSAGE( STATUS "\tURL: ${CURL_URL}" ) - include(FetchContent) - FetchContent_Declare( - CURL - URL ${CURL_URL} - ) - FetchContent_MakeAvailable(CURL) - SET(CURL_INCLUDE_DIRS = ${curl_SOURCE_DIR}/include) - SET(CURL_LIBRARIES libcurl) -ELSE() - MESSAGE( STATUS "\tInclude Directory: ${CURL_INCLUDE_DIRS}" ) - MESSAGE( STATUS "\tLibraries: ${CURL_LIBRARIES}" ) -ENDIF() \ No newline at end of file +fdpapi_add_external( + "CURL" + URL "https://github.com/curl/curl/archive/refs/tags/curl-7_80_0.zip" +) diff --git a/external/digestpp.cmake b/external/digestpp.cmake index 91b1d51..7762db2 100644 --- a/external/digestpp.cmake +++ b/external/digestpp.cmake @@ -1,12 +1,9 @@ -set(DIGESTCPP_URL "https://github.com/LiamPattinson/digestpp.git") -set(DIGESTCPP_COMMIT "6f6f134") - -message(STATUS "[DigestCPP]" ) - -message(STATUS "\tDigestCpp Will be installed.") -message(STATUS "\t(Using a fork with a CMake build system)") -message(STATUS "\tURL: ${DIGESTCPP_URL}") -message(STATUS "\tCOMMIT HASH: ${DIGESTCPP_COMMIT}") +fdpapi_add_external( + "DIGESTPP" + PKG_NAME "digestpp" + REPO "https://github.com/LiamPattinson/digestpp.git" + TAG "6f6f134" +) # windows.h will conflict with min functions in digestcpp # Because of macro definitions of min and max @@ -15,10 +12,3 @@ if(WIN32) add_definitions(-DNOMINMAX) add_definitions(-DNOGDI) endif() - -FetchContent_Declare( - DIGESTPP - GIT_REPOSITORY ${DIGESTCPP_URL} - GIT_TAG ${DIGESTCPP_COMMIT} -) -FetchContent_MakeAvailable(DIGESTPP) diff --git a/external/ghc.cmake b/external/ghc.cmake index e8bf803..c52d7c9 100644 --- a/external/ghc.cmake +++ b/external/ghc.cmake @@ -1,12 +1,6 @@ -set(GHC_URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip") - -message( STATUS "[GHC]" ) -message( STATUS "\tGHC Will be installed." ) -message( STATUS "\tURL: ${GHC_URL}" ) - set(GHC_FILESYSTEM_WITH_INSTALL ON CACHE INTERNAL "Create import targets for ghc-filesystem") -FetchContent_Declare( - GHC - URL ${GHC_URL} +fdpapi_add_external( + "GHC" + URL "https://github.com/gulrak/filesystem/archive/refs/tags/v1.5.10.zip" + PKG_NAME "ghc_filesystem" ) -FetchContent_MakeAvailable(GHC) diff --git a/external/jsoncpp.cmake b/external/jsoncpp.cmake index 913ab33..11c8e9c 100644 --- a/external/jsoncpp.cmake +++ b/external/jsoncpp.cmake @@ -1,14 +1,6 @@ -SET( JSONCPP_URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip" ) - -MESSAGE( STATUS "[JsonCPP]" ) -MESSAGE( STATUS "\tJsonCpp Will be installed." ) -MESSAGE( STATUS "\tURL: ${JSONCPP_URL}" ) - -SET (JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't build json-cpp tests") - -FetchContent_Declare( - JsonCpp - URL ${JSONCPP_URL} +set(JSONCPP_WITH_TESTS OFF CACHE INTERNAL "Don't build json-cpp tests") +fdpapi_add_external( + "JSONCPP" + URL "https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.5.zip" + PKG_NAME "jsoncpp" ) -FetchContent_MakeAvailable(JsonCpp) - diff --git a/external/toml11.cmake b/external/toml11.cmake index 715134f..cca2531 100644 --- a/external/toml11.cmake +++ b/external/toml11.cmake @@ -1,11 +1,5 @@ -SET( TOML11_URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" ) -MESSAGE( STATUS "[TOML 11]" ) - -MESSAGE( STATUS "\tTOML 11 Will be installed." ) -MESSAGE( STATUS "\tURL: ${TOML11_URL}" ) - -FetchContent_Declare( - toml11 - URL ${TOML11_URL} +fdpapi_add_external( + "TOML11" + URL "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.0.zip" + PKG_NAME "toml11" ) -FetchContent_MakeAvailable(toml11) diff --git a/external/yaml_cpp.cmake b/external/yaml_cpp.cmake index 0d67887..51c3db3 100644 --- a/external/yaml_cpp.cmake +++ b/external/yaml_cpp.cmake @@ -1,20 +1,12 @@ -set(YAML_CPP_GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git") -set(YAML_CPP_GIT_TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f") - -message(STATUS "[YAML-cpp]") -message(STATUS "\tYAML-cpp Will be installed.") -message(STATUS "\tURL: ${YAML_CPP_GIT_REPOSITORY}") -message(STATUS "\tCOMMIT HASH: ${YAML_CPP_GIT_TAG}") - set(YAML_CPP_BUILD_TESTS OFF CACHE INTERNAL "Build yaml-cpp tests") if(BUILD_SHARED_LIBS) set(YAML_BUILD_SHARED_LIBS ON CACHE INTERNAL "Build .so for yaml-cpp") endif() set(YAML_CPP_INSTALL ON CACHE INTERNAL "Include export targets for installation") -FetchContent_Declare( - yaml-cpp - GIT_REPOSITORY ${YAML_CPP_GIT_REPOSITORY} - GIT_TAG ${YAML_CPP_GIT_TAG} +fdpapi_add_external( + "YAMLCPP" + REPO "https://github.com/jbeder/yaml-cpp.git" + TAG "1b50109f7bea60bd382d8ea7befce3d2bd67da5f" + PKG_NAME "yaml-cpp" ) -FetchContent_MakeAvailable(yaml-cpp) diff --git a/include/fdp/fdp.h b/include/fdp/fdp.h new file mode 100644 index 0000000..d46bf11 --- /dev/null +++ b/include/fdp/fdp.h @@ -0,0 +1,202 @@ +#ifndef __FDP_C_API__ +#define __FDP_C_API__ + +#ifdef __cplusplus + +#include "fdp.hxx" + +extern "C" { + +#endif + +/** + * @brief Struct providing an interface to the pipeline. + * + * Defined in the implementation file, as it depends on C++ features. A pointer + * to this struct should be passed to functions in the C API. Set up by the + * function fdp_init, and finalised by fdp_finalise. Can also be generated from + * a C++ DataPipeline using to_c_struct. + */ +struct FdpDataPipeline; +typedef struct FdpDataPipeline FdpDataPipeline; + +/** + * @brief Enumeration used to denote different error types. + * + * The underlying C++ API will raise a number of different exception types. + * These map to integer error codes for C compatibility. + */ +enum FdpError { + FDP_ERR_NONE = 0, + FDP_ERR_CONFIG_PARSE = 1, + FDP_ERR_REST_API_QUERY = 2, + FDP_ERR_JSON_PARSE = 3, + FDP_ERR_VALIDATION = 4, + FDP_ERR_SYNC = 5, + FDP_ERR_WRITE = 6, + FDP_ERR_TOML = 7, + FDP_ERR_OTHER = 8 +}; +typedef enum FdpError FdpError; + +/** + * @brief Initialise the pipeline. + * + * Should be called once before any calls to fdp_link_read or fdp_link_write. If + * called more than once, returns FDP_ERR_OTHER. + * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. The + * user should declare a pointer to a FdpDataPipeline, and pass its address to + * this function. This function will then initialise the pipeline. + * + * @param config_file_path Path to the `config.yaml` file for this FDP run. + * Should be at the location `${FDP_CONFIG_DIR}/config.yaml`. + * + * @param script_file_path Path to the script which initiates this FDP run. + * Should be at the location `${FDP_CONFIG_DIR}/script.sh` on Linux/Mac or + * `${FDP_CONFIG_DIR}/script.bat` on Windows. + * + * @param token Token used to connect to FDP registry. May be set to `NULL`. + * + * @return Error code. + */ +FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, + const char *script_file_path, const char *token); + +/** + * @brief Finalise the pipeline. + * + * Must be called after a call to fdp_init. + * + * Record all data products and meta data to the registry. Update the code run + * with all appropriate meta data. + * + * @param data_pipeline Pointer-to-pointer of a FdpDataPipeline object. This + * function finalises the FdpDataPipeline, and sets its pointer to NULL. + * + * @return Error code. + */ +FdpError fdp_finalise(FdpDataPipeline **data_pipeline); + +/** + * @brief Set a path to a given data product while recording its meta data for + * the code run. + * + * Must be called after fdp_init and before fdp_finalise. + * + * @param data_pipeline Pointer to a FdpDataPipeline object. + * + * @param data_product Path to the input file. + * + * @param data_store_path Path to the assigned data store location. The user + * should allocate sufficient memory beforehand. + * + * @param data_store_path_len Size of the buffer used for the data store + * path. + * + * @return Error code + */ +FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *data_product, + char *data_store_path, size_t data_store_path_len); + +/** + * @brief Set a path to a given data product while recording its meta data for + * the code run. + * + * Must be called after fdp_init and before fdp_finalise. + * + * @param data_pipeline Pointer to a FdpDataPipeline object. + * + * @param data_product Path to the output file. + * + * @param data_store_path Path to the assigned data store location. The user + * should allocate sufficient memory beforehand. + * + * @param data_store_path_len Size of the buffer used for the data store + * path. + * + * @return Error code + */ +FdpError fdp_link_write(FdpDataPipeline *data_pipeline, + const char *data_product, char *data_store_path, size_t data_store_path_len); + +/** + * @brief Enumeration used to denote the different levels of logging. + * + * Each level of logging includes all log levels greater than it, so setting the + * log level to `DEBUG` will include all log types except `TRACE`. These + * correspond to the C++ logging levels `FairDataPipeline::logging::LOG_LEVEL`. + */ +enum FdpLogLevel { + FDP_LOG_TRACE = 0, + FDP_LOG_DEBUG = 1, + FDP_LOG_INFO = 2, + FDP_LOG_WARN = 3, + FDP_LOG_ERROR = 4, + FDP_LOG_CRITICAL = 5, + FDP_LOG_OFF = 6 +}; +typedef enum FdpLogLevel FdpLogLevel; + +/** + * @brief Set the log level. Must call `fdp_init` first. + * + * @param log_level + */ +void fdp_set_log_level(FdpLogLevel log_level); + +/** + * @brief Get the current log level. Must call fdp_init first. + * + * @return Log level + */ +FdpLogLevel fdp_get_log_level(); + +/** + * @brief Write a message to the log. + * + * This will be passed to the C++ logger, + * `FairDataPipeline::logger::get_logger()->level() << msg`, where + * `level` is one of `trace`, `debug`, `info`, `warn`, `error`, or `critical`. + * + * @param log_level The type of log message to write, e.g. FDP_LOG_INFO, + * FDP_LOG_ERROR. + * + * @param msg The message to be written to log. + * + * @return Error code. 1 if logging unsuccessful, 0 otherwise. + */ +int fdp_log(FdpLogLevel log_level, const char *msg); + +#ifdef __cplusplus + +} // close extern "C" + +namespace FairDataPipeline { + +/** + * @brief Convert data pipeline from the C API to one in the C++ API. + */ +DataPipeline::sptr from_c_struct(FdpDataPipeline *data_pipeline); + +/** + * @brief Convert data pipeline from the C++ API to one in the C API. + * + * If the pipeline is set up using the C++ method DataPipeline::construct, this + * may be used to generate a C-compatible struct. Note that this uses 'new' to + * allocate the returned pointer, so the user should 'delete_c_struct` after use + * to avoid memory leaks. It is not recommended to mix usage of the C and C++ + * APIs for init and finalise functions. + */ +FdpDataPipeline *to_c_struct(DataPipeline::sptr data_pipeline); + +/** + * @brief Function to clean up FdpDataPipeline created by to_c_struct + */ +void delete_c_struct(FdpDataPipeline *data_pipeline); + +} // namespace FairDataPipeline + +#endif + +#endif // __FDP_C_API__ diff --git a/include/fdp/objects/config.hxx b/include/fdp/objects/config.hxx index c83491b..1710233 100644 --- a/include/fdp/objects/config.hxx +++ b/include/fdp/objects/config.hxx @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/include/fdp/objects/metadata.hxx b/include/fdp/objects/metadata.hxx index 5069ce7..1494ce7 100644 --- a/include/fdp/objects/metadata.hxx +++ b/include/fdp/objects/metadata.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include "digestpp/digestpp.hpp" diff --git a/include/fdp/registry/api.hxx b/include/fdp/registry/api.hxx index fddea6f..f061c22 100644 --- a/include/fdp/registry/api.hxx +++ b/include/fdp/registry/api.hxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 937c5cd..c66a8a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ set(FDPAPI_SOURCE_FILES ../include/fdp/fdp.hxx + ../include/fdp/fdp.h ../include/fdp/exceptions.hxx ../include/fdp/objects/api_object.hxx ../include/fdp/objects/config.hxx @@ -12,6 +13,7 @@ set(FDPAPI_SOURCE_FILES ../include/fdp/utilities/logging.hxx ../include/fdp/utilities/semver.hxx ./fdp.cxx + ./fdp_c_api.cxx ./objects/api_object.cxx ./objects/config.cxx ./objects/distribution.cxx @@ -47,36 +49,21 @@ target_include_directories( $ ) -if(NOT Boost_FOUND) - - target_include_directories(fdpapi PRIVATE ${boostregex_SOURCE_DIR}/include) - target_link_directories(fdpapi PRIVATE ${boostregex_BINARY_DIR}) - -else() - - target_link_libraries(fdpapi PRIVATE Boost::regex ) - -endif() - - # Dependencies target_link_libraries(fdpapi PRIVATE toml11::toml11) target_link_libraries(fdpapi PRIVATE digestpp::digestpp) -target_link_libraries(fdpapi PRIVATE ${CURL_LIBRARIES}) +target_link_libraries(fdpapi PRIVATE CURL::libcurl) target_link_libraries(fdpapi PRIVATE yaml-cpp) +target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) if(BUILD_SHARED_LIBS) target_link_libraries(fdpapi PRIVATE jsoncpp_lib) else() target_link_libraries(fdpapi PRIVATE jsoncpp_static) endif() -# ghc_filesystem is now not included at top level -target_link_libraries(fdpapi PRIVATE ghcFilesystem::ghc_filesystem) - - # Set rules for installing targets if(FDPAPI_WITH_INSTALL) - message("Building With Install") + message(STATUS "Building install components") if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/lib ) set(CMAKE_INSTALL_BINDIR ${CMAKE_INSTALL_PREFIX}/bin ) diff --git a/src/fdp_c_api.cxx b/src/fdp_c_api.cxx new file mode 100644 index 0000000..078610e --- /dev/null +++ b/src/fdp_c_api.cxx @@ -0,0 +1,325 @@ +#include +#include +#include +#include +#include +#include + +#include "fdp/exceptions.hxx" +#include "fdp/fdp.h" +#include "fdp/fdp.hxx" +#include "fdp/utilities/logging.hxx" + +namespace FDP = FairDataPipeline; + +// Define FdpDataPipeline struct and conversion routines + +struct FdpDataPipeline { + FDP::DataPipeline::sptr _pipeline; +}; + +FDP::DataPipeline::sptr FDP::from_c_struct(FdpDataPipeline *data_pipeline) { + return data_pipeline->_pipeline; +} + +FdpDataPipeline *FDP::to_c_struct(FDP::DataPipeline::sptr data_pipeline) { + return new FdpDataPipeline{data_pipeline}; +} + +void FDP::delete_c_struct(FdpDataPipeline *data_pipeline) { + delete data_pipeline; +} + +/** + * @brief Utility method, calls exception-raising function and returns error + * codes + * + * Used to permit calls to C++ functions that may raise exceptions within a C + * environment. If any exceptions are thrown, these are caught and converted to + * an error code using the enum type FdpError. + * + * Many functions in the C++ API operate on a shared pointer to a + * FairDataPipeline::DataPipeline object. To call a function on this shared + * pointer using exception_to_err_code, use a lambda that captures the shared + * pointer. + * + * @param function The C++ function to call. Must not have a void return type. + * + * @param ret Parameter in which to store the result of the function call. + * + * @param args Args to pass to the function. + * + * @return Error code + * + * @see exception_to_err_code_void + */ +template +FdpError exception_to_err_code(Function &&function, Return &ret, + Args &&... args) { + try { + ret = std::forward(function)(std::forward(args)...); + return FDP_ERR_NONE; + } catch (const FDP::config_parsing_error &) { + return FDP_ERR_CONFIG_PARSE; + } catch (const FDP::rest_apiquery_error &) { + return FDP_ERR_REST_API_QUERY; + } catch (const FDP::json_parse_error &) { + return FDP_ERR_JSON_PARSE; + } catch (const FDP::validation_error &) { + return FDP_ERR_VALIDATION; + } catch (const FDP::sync_error &) { + return FDP_ERR_SYNC; + } catch (const FDP::write_error &) { + return FDP_ERR_WRITE; + } catch (const FDP::toml_error &) { + return FDP_ERR_TOML; + } catch (...) { + return FDP_ERR_OTHER; + } +} + +/** + * @brief Companion to exception_to_err_code for non-returning functions. + */ +template +FdpError exception_to_err_code_void(Function &&function, Args &&... args) { + int dummy; + return exception_to_err_code( + [&function, &args...]() -> int { + std::forward(function)(std::forward(args)...); + return 0; + }, + dummy); +} + +/** + * @brief Get error name from error code + */ +std::string error_name(FdpError err) { + std::string result; + switch (err) { + FDP_ERR_NONE: + result = "None"; + break; + FDP_ERR_CONFIG_PARSE: + result = "Config Parse"; + break; + FDP_ERR_REST_API_QUERY: + result = "REST API Query"; + break; + FDP_ERR_JSON_PARSE: + result = "JSON Parse"; + break; + FDP_ERR_VALIDATION: + result = "Validation"; + break; + FDP_ERR_SYNC: + result = "Sync"; + break; + FDP_ERR_WRITE: + result = "Write"; + break; + FDP_ERR_TOML: + result = "TOML"; + break; + default: + result = "Other"; + } + return result; +} + +// ================= +// init and finalise +// ================= + +FdpError fdp_init(FdpDataPipeline **data_pipeline, const char *config_file_path, + const char *script_file_path, const char *token) { + std::string token_str = (token == nullptr ? "" : token); + FDP::DataPipeline::sptr cpp_data_pipeline; + FdpError err = exception_to_err_code( + FDP::DataPipeline::construct, cpp_data_pipeline, + std::string(config_file_path), std::string(script_file_path), token_str); + if (err) { + // Trust that the C++ API logged the error before throwing + *data_pipeline = nullptr; + return err; + } + *data_pipeline = FDP::to_c_struct(cpp_data_pipeline); + return err; +} + +FdpError fdp_finalise(FdpDataPipeline **data_pipeline) { + if (*data_pipeline == nullptr || (*data_pipeline)->_pipeline == nullptr) { + FDP::logger::get_logger()->error() + << "Pipeline not initialiased in call to fdp_finalise"; + return FDP_ERR_OTHER; + } + FdpError err = exception_to_err_code_void( + [](FDP::DataPipeline::sptr pipeline) { pipeline->finalise(); }, + (*data_pipeline)->_pipeline); + if (err) { + // Trust that the C++ API logged the error before throwing + return err; + } + FDP::delete_c_struct(*data_pipeline); + *data_pipeline = nullptr; + return err; +} + +template +FdpError _fdp_link(LinkFunction &&link_function, + const std::string &link_function_name, + FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + // Ensure pipeline is initialised + if (data_pipeline == nullptr || data_pipeline->_pipeline == nullptr) { + FDP::logger::get_logger()->error() + << " Data pipeline not initialised in call to " << link_function_name; + return FDP_ERR_OTHER; + } + // Ensure input and output paths are valid + if (path == nullptr) { + FDP::logger::get_logger()->error() + << "Input path is NULL in call to " << link_function_name; + return FDP_ERR_OTHER; + } + if (output == nullptr) { + FDP::logger::get_logger()->error() + << "Output path is NULL in call to " << link_function_name; + return FDP_ERR_OTHER; + } + // Check output len is valid + if (output_len == 0) { + FDP::logger::get_logger()->error() + << "output_len is zero in call to " << link_function_name; + return FDP_ERR_OTHER; + } + // Check input is null terminated, max 4096 chars, including terminator + // TODO Should we check MAX_PATH/PATH_MAX here? + bool path_null_terminated = false; + std::size_t path_len = 0; + for (std::size_t ii = 0; ii < 4096; ++ii) { + if (path[ii] == '\0') { + path_null_terminated = true; + path_len = ii; + break; + } + } + if (!path_null_terminated) { + FDP::logger::get_logger()->error() + << "Input path is not null-terminated or is longer than 4095 chars in " + "call to " + << link_function_name; + return FDP_ERR_OTHER; + } + // Check input and output don't overlap + auto x1 = reinterpret_cast(path); + auto x2 = x1 + path_len; + auto y1 = reinterpret_cast(output); + auto y2 = y1 + output_len; + if (std::max(x1, y1) <= std::min(x2, y2)) { + FDP::logger::get_logger()->error() + << "Input and output paths overlap in call to " << link_function_name; + return FDP_ERR_OTHER; + } + // Use C++ strings over C strings to interface with the C++ pipeline + std::string input_path = path; + std::string output_path; + // Call either link_read or link_write on the pipeline, sets output_path + FdpError err = + exception_to_err_code(std::forward(link_function), + output_path, data_pipeline->_pipeline, input_path); + if (err) { + // Trust that the C++ API logged the error before throwing + return err; + } + // Don't copy if output_path won't fit in output buffer + // Use >= instead of > to account for null terminator + if (output_path.size() >= output_len) { + FDP::logger::get_logger()->error() + << "Output path won't fit in buffer in call to " << link_function_name; + return FDP_ERR_OTHER; + } + strncpy(output, output_path.c_str(), output_len); + return FDP_ERR_NONE; +} + +FdpError fdp_link_read(FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + return _fdp_link( + [](FDP::DataPipeline::sptr pipeline, std::string &path) -> std::string { + return pipeline->link_read(path); + }, + "fdp_link_read", data_pipeline, path, output, output_len); +} + +FdpError fdp_link_write(FdpDataPipeline *data_pipeline, const char *path, + char *output, size_t output_len) { + return _fdp_link( + [](FDP::DataPipeline::sptr pipeline, std::string &path) -> std::string { + return pipeline->link_write(path); + }, + "fdp_link_write", data_pipeline, path, output, output_len); +} + +// ======= +// logging +// ======= + +/** + * @brief Map converting C API logging enums to the C++ API + */ +std::map to_cpp_enum = { + {FDP_LOG_TRACE, FDP::logging::TRACE}, + {FDP_LOG_DEBUG, FDP::logging::DEBUG}, + {FDP_LOG_INFO, FDP::logging::INFO}, + {FDP_LOG_WARN, FDP::logging::WARN}, + {FDP_LOG_ERROR, FDP::logging::ERROR}, + {FDP_LOG_CRITICAL, FDP::logging::CRITICAL}, + {FDP_LOG_OFF, FDP::logging::OFF}}; + +/** + * @brief Map converting C++ API logging enums to the C API + */ +std::map to_c_enum = { + {FDP::logging::TRACE, FDP_LOG_TRACE}, + {FDP::logging::DEBUG, FDP_LOG_DEBUG}, + {FDP::logging::INFO, FDP_LOG_INFO}, + {FDP::logging::WARN, FDP_LOG_WARN}, + {FDP::logging::ERROR, FDP_LOG_ERROR}, + {FDP::logging::CRITICAL, FDP_LOG_CRITICAL}, + {FDP::logging::OFF, FDP_LOG_OFF}}; + +void fdp_set_log_level(FdpLogLevel log_level) { + FDP::logger::get_logger()->set_level(to_cpp_enum[log_level]); +} + +FdpLogLevel fdp_get_log_level() { + return to_c_enum[FDP::logger::get_logger()->get_level()]; +} + +int fdp_log(FdpLogLevel log_level, const char *msg) { + switch (log_level) { + case FDP_LOG_TRACE: + FDP::logger::get_logger()->trace() << msg; + break; + case FDP_LOG_DEBUG: + FDP::logger::get_logger()->debug() << msg; + break; + case FDP_LOG_INFO: + FDP::logger::get_logger()->info() << msg; + break; + case FDP_LOG_WARN: + FDP::logger::get_logger()->warn() << msg; + break; + case FDP_LOG_ERROR: + FDP::logger::get_logger()->error() << msg; + break; + case FDP_LOG_CRITICAL: + FDP::logger::get_logger()->critical() << msg; + break; + default: + return 1; + } + return 0; +} diff --git a/src/objects/config.cxx b/src/objects/config.cxx index 8681b9b..08b1e0c 100644 --- a/src/objects/config.cxx +++ b/src/objects/config.cxx @@ -240,14 +240,14 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { // Remove the Write Data Store from config file path Json::Value config_storage_location_value_; - if(config_file_path_.string().find(meta_data_()["write_data_store"].as()) !=std::string::npos){ - config_storage_location_value_["path"] = config_file_path_.string().replace( - config_file_path_.string().find(meta_data_()["write_data_store"].as()), - sizeof(meta_data_()["write_data_store"].as()) - 1, ""); - } - else { - config_storage_location_value_["path"] = config_file_path_.string(); + config_storage_location_value_["path"] = config_file_path_.string(); + std::size_t ind = config_file_path_.string().find(meta_data_()["write_data_store"].as()); + if(ind != std::string::npos){ + config_storage_location_value_["path"] = config_storage_location_value_["path"].asString().erase( + ind, meta_data_()["write_data_store"].as().length() + ); } + config_storage_location_value_["path"] = remove_backslash_from_path(config_storage_location_value_["path"].asString()); config_storage_location_value_["path"] = API::remove_leading_forward_slash(config_storage_location_value_["path"].asString()); config_storage_location_value_["public"] = true; @@ -280,16 +280,15 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { this->config_obj_ = ApiObject::from_json( j_config_obj ); - Json::Value script_storage_location_value_; - - if(script_file_path_.string().find(meta_data_()["write_data_store"].as()) !=std::string::npos){ - script_storage_location_value_["path"] = script_file_path_.string().replace( - script_file_path_.string().find(meta_data_()["write_data_store"].as()), - sizeof(meta_data_()["write_data_store"].as()) - 1, ""); - } - else { - script_storage_location_value_["path"] = script_file_path_.string(); + Json::Value script_storage_location_value_; + script_storage_location_value_["path"] = script_file_path_.string(); + ind = script_file_path_.string().find(meta_data_()["write_data_store"].as()); + if(ind != std::string::npos){ + script_storage_location_value_["path"] = script_storage_location_value_["path"].asString().erase( + ind, meta_data_()["write_data_store"].as().length() + ); } + script_storage_location_value_["path"] = remove_backslash_from_path(script_storage_location_value_["path"].asString()); script_storage_location_value_["path"] = API::remove_leading_forward_slash(script_storage_location_value_["path"].asString()); script_storage_location_value_["hash"] = calculate_hash_from_file(script_file_path_); @@ -327,7 +326,7 @@ void FairDataPipeline::Config::initialise(RESTAPI api_location) { Json::Value j_code_repo_root = api_->post("storage_root", repo_storage_root_value_, token_); this->code_repo_storage_root_ = ApiObject::from_json( j_code_repo_root ); - std::string repo_storage_path_ = boost::regex_replace(meta_data_()["remote_repo"].as(), boost::regex(repo_storage_root_value_["root"].asString()), ""); + std::string repo_storage_path_ = std::regex_replace(meta_data_()["remote_repo"].as(), std::regex(repo_storage_root_value_["root"].asString()), ""); Json::Value repo_storage_location_value_; repo_storage_location_value_["hash"] = meta_data_()["latest_commit"].as(); diff --git a/src/objects/metadata.cxx b/src/objects/metadata.cxx index 6993855..6860aa2 100644 --- a/src/objects/metadata.cxx +++ b/src/objects/metadata.cxx @@ -64,11 +64,11 @@ std::string current_time_stamp(bool file_name) { } std::string remove_local_from_root(const std::string &root){ - return boost::regex_replace(root, boost::regex(std::string("file:\\/\\/")), ""); + return std::regex_replace(root, std::regex(std::string("file:\\/\\/")), ""); } std::string remove_backslash_from_path(const std::string &path){ - return boost::regex_replace(path, boost::regex(std::string("\\\\")), "/"); + return std::regex_replace(path, std::regex(std::string("\\\\")), "/"); } bool file_exists( const std::string &Filename ) @@ -84,4 +84,4 @@ std::string read_token(const ghc::filesystem::path &token_path){ return key_str_; } -}; // namespace FairDataPipeline \ No newline at end of file +}; // namespace FairDataPipeline diff --git a/src/registry/api.cxx b/src/registry/api.cxx index 6f2dd53..b3bc638 100644 --- a/src/registry/api.cxx +++ b/src/registry/api.cxx @@ -95,7 +95,7 @@ CURL *API::setup_download_session_(const ghc::filesystem::path &addr_path, Json::Value API::get_request(const ghc::filesystem::path &addr_path, long expected_response, std::string token) { - std::string addr_path_ = boost::regex_replace(addr_path.string(), boost::regex(std::string("\\\\")), "/"); + std::string addr_path_ = std::regex_replace(addr_path.string(), std::regex(std::string("\\\\")), "/"); return get_request(addr_path_, expected_response); } @@ -180,16 +180,16 @@ std::string API::json_to_query_string(Json::Value &json_value) { // add the key and value to the return string after removing the api // address with regex rtn += key + "=" + - boost::regex_replace(json_value.get(key, "")[i].asString(), - boost::regex(regex_string), "$3") + + std::regex_replace(json_value.get(key, "")[i].asString(), + std::regex(regex_string), "$3") + "&"; } } else { // if it's not an array add the key and value to the return string after // removing the api address with regex rtn += key + "=" + - boost::regex_replace(json_value.get(key, "").asString(), - boost::regex(regex_string), "$3") + + std::regex_replace(json_value.get(key, "").asString(), + std::regex(regex_string), "$3") + "&"; } } @@ -200,7 +200,7 @@ std::string API::json_to_query_string(Json::Value &json_value) { std::string API::escape_space(std::string &str) { // Using regex replace space with html character (%20) - return std::string(boost::regex_replace(str, boost::regex(" "), "%20")); + return std::string(std::regex_replace(str, std::regex(" "), "%20")); } Json::Value API::post(std::string addr_path, Json::Value &post_data, @@ -222,7 +222,7 @@ Json::Value API::post_file_type(Json::Value &post_data, const std::string &token logger::get_logger()->error() << "Error: Post Data does not contain a file extension"; throw rest_apiquery_error("Failed to post file_type"); } - post_data["extension"] = boost::regex_replace(post_data["extension"].asString(), boost::regex("."), ""); + post_data["extension"] = regex_replace(post_data["name"].asString(), std::regex("."), ""); Json::Value _file_type_query; _file_type_query["extension"] = post_data["extension"]; Json::Value _file_type_exists = get_by_json_query("file_type", _file_type_query); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4ef7830..4bb99ab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -63,9 +63,8 @@ target_link_libraries(fdpapi-tests PRIVATE fdpapi::fdpapi) target_link_libraries(fdpapi-tests PRIVATE gtest gtest_main) target_link_libraries(fdpapi-tests PRIVATE toml11::toml11) -target_link_libraries(fdpapi-tests PRIVATE digestpp) -target_link_libraries(fdpapi-tests PRIVATE boost_regex) -target_link_libraries(fdpapi-tests PRIVATE ${CURL_LIBRARIES}) +target_link_libraries(fdpapi-tests PRIVATE digestpp::digestpp) +target_link_libraries(fdpapi-tests PRIVATE CURL::libcurl) target_link_libraries(fdpapi-tests PRIVATE yaml-cpp) if(BUILD_SHARED_LIBS) target_link_libraries(fdpapi-tests PRIVATE jsoncpp_lib) diff --git a/test/data/read_csv.yaml b/test/data/read_csv.yaml index 09eb983..cd11b56 100644 --- a/test/data/read_csv.yaml +++ b/test/data/read_csv.yaml @@ -13,5 +13,8 @@ run_metadata: read: - data_product: test/csv + use: + version: 0.0.1 +- data_product: test/csv/c use: version: 0.0.1 \ No newline at end of file diff --git a/test/data/write_csv.yaml b/test/data/write_csv.yaml index 1b42fea..b941968 100644 --- a/test/data/write_csv.yaml +++ b/test/data/write_csv.yaml @@ -14,6 +14,11 @@ run_metadata: write: - data_product: test/csv + description: test csv file with simple data + file_type: csv + use: + version: 0.0.1 +- data_product: test/csv/c description: test csv file with simple data file_type: csv use: diff --git a/test/test_c_api.cxx b/test/test_c_api.cxx new file mode 100644 index 0000000..d1ac2c7 --- /dev/null +++ b/test/test_c_api.cxx @@ -0,0 +1,145 @@ +#ifndef TESTDIR +#define TESTDIR "" +#endif + +#include +#include +#include +#include + +#include "fdp/fdp.h" +#include "fdp/objects/metadata.hxx" // read_token + +#include "gtest/gtest.h" +#include + +namespace fs = ghc::filesystem; +namespace fdp = FairDataPipeline; + +std::string home_dir() { + std::string home; +#ifdef _WIN32 + home = getenv("HOMEDRIVE"); + home += getenv("HOMEPATH"); +#else + home = getenv("HOME"); +#endif + return home; +} + +#define BUFFER_SIZE 512 + +TEST(CTest, link_read_write) { + fdp_set_log_level(FDP_LOG_DEBUG); + char buf[BUFFER_SIZE]; + + // Initialise + FdpDataPipeline *pipeline; + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + ASSERT_EQ(fdp_init(&pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); + + // Test link write + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_write(pipeline, "test/csv/c", buf, BUFFER_SIZE), + FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Write to new path + std::ofstream fstream(buf); + fstream << "Test"; + fstream.close(); + + // Finalise and re-initialise + ASSERT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); + config = fs::path(TESTDIR) / "data" / "read_csv.yaml"; + ASSERT_EQ(fdp_init(&pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); + + // Test link read + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_read(pipeline, "test/csv/c", buf, BUFFER_SIZE), + FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Finalise again + EXPECT_EQ(fdp_finalise(&pipeline), FDP_ERR_NONE); +} + +TEST(CTest, cpp_to_c) { + fdp_set_log_level(FDP_LOG_DEBUG); + char buf[512]; + + // Initialise using C++ + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + auto cpp_pipeline = + fdp::DataPipeline::construct(config.string(), script.string(), token); + + // Switch to C API, use temporary FdpDataPipeline + FdpDataPipeline *c_pipeline = fdp::to_c_struct(cpp_pipeline); + + // Test link write + buf[0] = '\0'; // Ensure strlen of output buffer is 0 + EXPECT_EQ(fdp_link_write(c_pipeline, "test/csv", buf, BUFFER_SIZE), + FDP_ERR_NONE); + EXPECT_GT(strlen(buf), 1); + + // Write to new path + std::ofstream fstream(buf); + fstream << "Test"; + fstream.close(); + + // Finish working in C, delete FdpDataPipeline + fdp::delete_c_struct(c_pipeline); + + // Finalise in C++ + cpp_pipeline->finalise(); +} + +TEST(CTest, c_to_cpp) { + fdp_set_log_level(FDP_LOG_DEBUG); + + // Initialise using C + FdpDataPipeline *c_pipeline; + fs::path config = fs::path(TESTDIR) / "data" / "write_csv.yaml"; + fs::path script = fs::path(TESTDIR) / "test_script.sh"; + std::string token = + fdp::read_token(fs::path(home_dir()) / ".fair" / "registry" / "token"); + ASSERT_EQ(fdp_init(&c_pipeline, config.string().c_str(), + script.string().c_str(), token.c_str()), + FDP_ERR_NONE); + + // Switch to C++ API + auto cpp_pipeline = fdp::from_c_struct(c_pipeline); + + // Test link write + std::string data_product = "test/csv"; + fs::path currentLink = fs::path(cpp_pipeline->link_write(data_product)); + EXPECT_GT(currentLink.string().size(), 1); + + // Write to new path + std::ofstream fstream(currentLink); + fstream << "Test"; + fstream.close(); + + // Finish working in C++ + cpp_pipeline = nullptr; + + // Finalise in C + EXPECT_EQ(fdp_finalise(&c_pipeline), FDP_ERR_NONE); +} + +TEST(CTest, log_levels) { + fdp_set_log_level(FDP_LOG_INFO); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_INFO); + fdp_set_log_level(FDP_LOG_DEBUG); + EXPECT_EQ(fdp_get_log_level(), FDP_LOG_DEBUG); +}