diff --git a/CMakeLists.txt b/CMakeLists.txt index f795c0a65..93a62a46e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,7 @@ find_package(OpenSSL 1.0.0 REQUIRED) find_package(PkgConfig REQUIRED) find_package(spdlog REQUIRED) find_package(Threads REQUIRED) +find_package(nlohmann_json REQUIRED) find_package(OpenMP) find_package(IBVerbs) find_package(RDMACM) diff --git a/common/include/villas/config.hpp.in b/common/include/villas/config.hpp.in index 8babceb18..3325b794a 100644 --- a/common/include/villas/config.hpp.in +++ b/common/include/villas/config.hpp.in @@ -40,6 +40,7 @@ // Library features #cmakedefine FMT_LEGACY_OSTREAM_FORMATTER +#cmakedefine WITH_CONFIG #cmakedefine WITH_GHC_FS // clang-format on diff --git a/common/include/villas/json.hpp b/common/include/villas/json.hpp new file mode 100644 index 000000000..a8bcd1285 --- /dev/null +++ b/common/include/villas/json.hpp @@ -0,0 +1,47 @@ +/* Json parsing and support code. + * + * Author: Philipp Jungkamp + * SPDX-FileCopyrightText: 2024-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +#include + +extern template class nlohmann::basic_json<>; + +namespace villas { + +using Json = nlohmann::json; +using JsonPointer = Json::json_pointer; + +// forward declaration for villas/jansson.hpp compatibility header +class JanssonPtr; +void to_json(Json &json, JanssonPtr const &jansson); +void from_json(Json const &json, JanssonPtr &jansson); + +struct LoadConfigFileOptions { + bool allow_libconfig = false; + bool allow_environment = false; + bool allow_include = false; +}; + +// load a configuration file +Json load_config_file(fs::path const &path, LoadConfigFileOptions const &opts); + +}; // namespace villas + +// forward declaration for libjansson's json_t +struct json_t; + +// convert borrowed libjansson +void to_json(villas::Json &json, json_t const *jansson); + +template <> // format nlohmann::json using operator<< +struct fmt::formatter : ostream_formatter {}; + +template <> // format json_pointer using operator<< +struct fmt::formatter : ostream_formatter {}; diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index 318aeceac..0919013bf 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -131,6 +131,10 @@ template struct overloaded : Ts... { // Explicit deduction guide (not needed as of C++20) template overloaded(Ts...) -> overloaded; +// glob-style filesystem pattern matching +std::vector glob(fs::path const &pattern, + std::span searchDirectories); + void write_to_file(std::string data, const fs::path file); namespace base64 { diff --git a/common/lib/CMakeLists.txt b/common/lib/CMakeLists.txt index 497fe9994..816df3a42 100644 --- a/common/lib/CMakeLists.txt +++ b/common/lib/CMakeLists.txt @@ -15,6 +15,7 @@ add_library(villas-common SHARED cpuset.cpp dsp/pid.cpp hist.cpp + json.cpp kernel/kernel.cpp kernel/rt.cpp list.cpp @@ -55,6 +56,7 @@ target_include_directories(villas-common PUBLIC ) target_link_libraries(villas-common PUBLIC + nlohmann_json::nlohmann_json PkgConfig::JANSSON PkgConfig::UUID ${OPENSSL_LIBRARIES} diff --git a/common/lib/json.cpp b/common/lib/json.cpp new file mode 100644 index 000000000..689bb8ced --- /dev/null +++ b/common/lib/json.cpp @@ -0,0 +1,405 @@ +/* Json configuration parsing implementation. + * + * Author: Philipp Jungkamp + * SPDX-FileCopyrightText: 2024-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#ifdef WITH_CONFIG +#include +#endif + +template class nlohmann::basic_json<>; + +using namespace std::string_view_literals; + +void to_json(villas::Json &json, ::json_t const *jansson) { + switch (json_typeof(jansson)) { + using enum json_type; + + case JSON_ARRAY: { + std::size_t index; + ::json_t const *value; + + json = villas::Json::array(); + json_array_foreach (jansson, index, value) + json.push_back(value); + } break; + + case JSON_OBJECT: { + char const *key; + ::json_t const *value; + + json = villas::Json::object(); + // The const_cast below is safe as long as we don't replace the contained value + // by accessing the underlying iterator with json_object_key_to_iter(key) and + // json_object_iter_set or json_object_iter_set_new. + // + // There is no API in libjansson to iterate or even enumerate keys of a const object. + // + // See https://github.com/akheron/jansson/issues/578 + json_object_foreach (const_cast<::json_t *>(jansson), key, value) + json.emplace(key, value); + } break; + + case JSON_STRING: { + json = std::string{json_string_value(jansson), json_string_length(jansson)}; + } break; + + case JSON_INTEGER: { + json = json_integer_value(jansson); + } break; + + case JSON_REAL: { + json = json_real_value(jansson); + } break; + + case JSON_TRUE: { + json = true; + } break; + + case JSON_FALSE: { + json = false; + } break; + + case JSON_NULL: + default: { + json = nullptr; + } break; + } +} + +namespace villas { + +void from_json(Json const &json, JanssonPtr &jansson) { + switch (json.type()) { + using value_t = Json::value_t; + + case value_t::array: { + jansson.reset(json_array()); + for (auto const &item : json) + json_array_append_new(jansson.get(), item.get().release()); + } break; + + case value_t::object: { + jansson.reset(json_object()); + for (auto const &[key, value] : json.items()) + json_object_set_new_nocheck(jansson.get(), key.c_str(), + value.get().release()); + } break; + + case value_t::string: { + auto const string = json.get(); + jansson.reset(json_string_nocheck(string->c_str())); + } break; + + case value_t::number_integer: { + auto const integer = json.get(); + jansson.reset(json_integer(*integer)); + } break; + + case value_t::number_unsigned: { + auto const integer = json.get(); + jansson.reset(json_integer(*integer)); + } break; + + case value_t::number_float: { + auto const real = json.get(); + jansson.reset(json_real(*real)); + } break; + + case value_t::boolean: { + auto const boolean = json.get(); + jansson.reset(json_boolean(*boolean)); + } break; + + case value_t::null: { + jansson.reset(json_null()); + } break; + + case value_t::binary: + throw std::runtime_error{"cannot convert binary value to jansson"}; + + case value_t::discarded: + throw std::runtime_error{"cannot convert discarded value to jansson"}; + + default: + __builtin_unreachable(); + } +} + +void to_json(Json &json, JanssonPtr const &jansson) { + to_json(json, jansson.get()); +} + +namespace { + +// implementation of deprecated variable substitutions +void expand_substitutions(Json &value, bool resolve_env, + fs::path const *include_dir) { + if (not value.is_string()) + return; + + if (not resolve_env and not include_dir) + return; + + constexpr static auto DEPRECATED_INCLUDE_KEYWORD = "@include "sv; + auto logger = Log::get("config"); + auto string = value.get_ref(); + auto do_include = false; + auto expanded = std::size_t{0}; + + // check for legacy @include keyword + if (include_dir and string.starts_with(DEPRECATED_INCLUDE_KEYWORD)) { + do_include = true; + expanded = DEPRECATED_INCLUDE_KEYWORD.length(); + } + + // legacy environment variable substitution syntax + static auto const env_regex = std::regex(R"--(\$\{([^}]+)\})--"); + enum : std::size_t { + CAPTURE_ALL = 0, + CAPTURE_NAME, + }; + + // expand deprecated environment substition syntax + std::smatch match; + while (resolve_env and std::regex_search(string.cbegin() + expanded, + string.cend(), match, env_regex)) { + auto name = std::string(match[CAPTURE_NAME]); + auto env_ptr = std::getenv(name.c_str()); + if (not env_ptr) + throw std::runtime_error( + fmt::format("Could substitute environment variable {:?}", name)); + + auto env_value = std::string_view(env_ptr); + auto [begin, end] = std::pair(match[CAPTURE_ALL]); + string.replace(begin, end, env_value.begin(), env_value.end()); + expanded += match.position() + env_value.length(); + } + + // expand deprecated @include directive + if (do_include) { + auto pattern = + std::string_view(string).substr(DEPRECATED_INCLUDE_KEYWORD.length()); + auto result = Json(nullptr); + for (auto path : utils::glob(pattern, std::span(include_dir, 1))) { + auto partial_result = + load_config_file(path, { + .allow_libconfig = false, + .allow_environment = resolve_env, + .allow_include = include_dir != nullptr, + }); + if (result.is_null()) + result = partial_result; + else if (partial_result.is_object() and result.is_object()) + result.update(partial_result, true); + else if (partial_result.is_array() and result.is_array()) + result.insert(result.end(), partial_result.begin(), + partial_result.end()); + } + + logger->warn("Found deprecated @include directive: {}", value); + value = std::move(result); + } else if (expanded) { + logger->warn("Found deprecated environment substitution: {}", value); + value = std::move(string); + } +} + +#ifdef WITH_CONFIG + +Json parse_libconfig_setting(::config_setting_t const *setting, + bool resolve_env, fs::path const *include_dir) { + switch (config_setting_type(setting)) { + case CONFIG_TYPE_ARRAY: + case CONFIG_TYPE_LIST: { + auto array = Json::array(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + array.push_back(parse_libconfig_setting(elem, resolve_env, include_dir)); + } + + return array; + } + + case CONFIG_TYPE_GROUP: { + auto object = Json::object(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + auto name = std::string(config_setting_name(elem)); + object.emplace(std::move(name), + parse_libconfig_setting(elem, resolve_env, include_dir)); + } + + return object; + } + + case CONFIG_TYPE_STRING: { + auto json = Json(std::string(config_setting_get_string(setting))); + expand_substitutions(json, resolve_env, include_dir); + return json; + } + + case CONFIG_TYPE_INT: { + return config_setting_get_int(setting); + } + + case CONFIG_TYPE_INT64: { + return std::int64_t{config_setting_get_int64(setting)}; + } + + case CONFIG_TYPE_FLOAT: { + return config_setting_get_float(setting); + } + + case CONFIG_TYPE_BOOL: { + return static_cast(config_setting_get_bool(setting)); + } + + case CONFIG_TYPE_NONE: + default: { + return nullptr; + } + } +} + +#if (LIBCONFIG_VER_MAJOR > 1) || \ + ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) + +struct LibconfigHook { + bool resolve_env; + fs::path const *include_dir; +}; + +extern "C" char const **libconfig_include_func(::config_t *config, char const *, + char const *pattern, + char const **error) noexcept { + auto hook = static_cast(config_get_hook(config)); + auto paths = std::vector{}; + + if (not hook->include_dir) { + *error = "include directives are disabled"; + return nullptr; + } + + auto pattern_json = Json(pattern); + expand_substitutions(pattern_json, hook->resolve_env, nullptr); + auto const &pattern_expanded = pattern_json.get_ref(); + + try { + paths = utils::glob(pattern_expanded, std::span(hook->include_dir, 1)); + std::erase_if(paths, [](auto const &path) { + auto ec = std::error_code{}; + return not fs::is_regular_file(path, ec); + }); + } catch (...) { + } + + if (paths.empty()) { + *error = "include directive did not match any file"; + return nullptr; + } + + auto ret = + static_cast(std::calloc(paths.size() + 1, sizeof(char *))); + auto index = std::size_t{0}; + for (auto &path : paths) + ret[index++] = strdup(path.c_str()); + + return ret; +} + +#endif + +Json load_libconfig_file(std::FILE *file, bool resolve_env, + fs::path const *include_dir) { + using ConfigDestroy = decltype([](::config_t *c) { ::config_destroy(c); }); + using ConfigGuard = std::unique_ptr<::config_t, ConfigDestroy>; + + ::config_t config; + ::config_init(&config); + auto guard = ConfigGuard(&config); + +#if (LIBCONFIG_VER_MAJOR > 1) || \ + ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) + auto hook = LibconfigHook{ + .resolve_env = resolve_env, + .include_dir = include_dir, + }; + + ::config_set_hook(&config, &hook); + ::config_set_include_func(&config, &libconfig_include_func); +#else + auto logger = Log::get("config"); + logger->warn("Loading configuration using an outdated version of libconfig"); + logger->warn( + "@include directives may not work as expected and can't be disabled"); + if (include_dir) + ::config_set_include_dir(&config, include_dir->c_str()); +#endif + if (auto ret = ::config_read(&config, file); ret != CONFIG_TRUE) { + throw std::runtime_error(fmt::format("Failed to load libconfig file: {}", + config_error_text(&config))); + } + + return parse_libconfig_setting(config_root_setting(&config), resolve_env, + include_dir); +} + +#endif // WITH_CONFIG + +} // namespace + +Json load_config_file(fs::path const &path, LoadConfigFileOptions const &opts) { + using FileDeleter = decltype([](std::FILE *c) { std::fclose(c); }); + using FilePtr = std::unique_ptr; + + auto file = FilePtr(std::fopen(path.c_str(), "r")); + if (not file) { + throw std::runtime_error( + fmt::format("Failed to open config file {}", path.string())); + } + + auto include_dir = path.parent_path(); + if (auto ext = path.extension(); ext == ".json") { + auto parser_callback = [&](int depth, Json::parse_event_t event, + Json &value) { + if (event == Json::parse_event_t::value) + expand_substitutions(value, opts.allow_environment, + opts.allow_include ? &include_dir : nullptr); + + return true; + }; + + return Json::parse(file.get(), parser_callback); + } else if (opts.allow_libconfig) { +#ifdef WITH_CONFIG + return load_libconfig_file(file.get(), opts.allow_environment, + opts.allow_include ? &include_dir : nullptr); +#else + throw std::runtime_error( + "VILLASnode was built without support for libconfig"); +#endif // WITH_CONFIG + } else { + throw std::runtime_error(fmt::format( + "Failed to load config file with unknown extension {}", ext.string())); + } +} + +} // namespace villas diff --git a/common/lib/utils.cpp b/common/lib/utils.cpp index 2bbeda096..848c607da 100644 --- a/common/lib/utils.cpp +++ b/common/lib/utils.cpp @@ -12,13 +12,14 @@ #include #include #include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -352,6 +353,70 @@ bool isPrivileged() { return true; } +// internal glob implementation details +namespace { +bool isGlobPattern(fs::path const &path) { + static const auto specialCharacters = fs::path("?*[").native(); + auto const &string = path.native(); + return std::ranges::find_first_of(string, specialCharacters) != string.end(); +} + +bool isGlobMatch(fs::path const &pattern, fs::path const &path) { + return ::fnmatch(pattern.c_str(), path.c_str(), FNM_PATHNAME) == 0; +} + +void globImpl(std::vector &result, fs::path &&path, + std::ranges::subrange pattern) { + [[maybe_unused]] auto discardErrorCode = std::error_code{}; + + if (pattern.empty()) { + // we've reached the end of our pattern + if (fs::exists(path, discardErrorCode)) + result.push_back(path); + return; + } + + if (not fs::is_directory(path, discardErrorCode)) + return; + + if (not isGlobPattern(pattern.front())) { + path /= pattern.front(); + return globImpl(result, std::move(path), std::move(pattern).next()); + } else { + auto nextPattern = pattern.next(); + for (auto entry : fs::directory_iterator(path)) { + if (not isGlobMatch(pattern.front(), entry.path().filename())) + continue; + + globImpl(result, fs::path(entry.path()), nextPattern); + } + } +} +} // namespace + +std::vector glob(fs::path const &pattern, + std::span searchDirectories) { + auto logger = Log::get("glob"); + std::vector result; + if (pattern.is_absolute()) { + logger->debug("Matching absolute pattern {:?}", pattern.string()); + globImpl(result, pattern.root_path(), pattern); + } else { + for (auto path : searchDirectories) { + logger->debug("Matching relative pattern {:?} in {:?}", pattern.string(), + path.string()); + globImpl(result, std::move(path), pattern); + } + } + + if (result.empty()) { + throw std::runtime_error( + fmt::format("Could not find any file matching {:?}", pattern.string())); + } + + return result; +} + void write_to_file(std::string data, const fs::path file) { villas::Log::get("Filewriter")->debug("{} > {}", data, file.string()); std::ofstream outputFile(file.string()); diff --git a/include/villas/config_class.hpp b/include/villas/config_class.hpp deleted file mode 100644 index e6b372b9c..000000000 --- a/include/villas/config_class.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* Configuration file parsing. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include - -#include -#include - -#include -#include -#include - -#ifdef WITH_CONFIG -#include -#endif - -namespace villas { -namespace node { - -class Config { - -protected: - using str_walk_fcn_t = std::function; - - Logger logger; - - std::list includeDirectories; - std::string configPath; - - // Check if file exists on local system. - static bool isLocalFile(const std::string &uri) { - return access(uri.c_str(), F_OK) != -1; - } - - // Decode configuration file. - json_t *decode(FILE *f); - -#ifdef WITH_CONFIG - // Convert libconfig .conf file to libjansson .json file. - json_t *libconfigDecode(FILE *f); - - static const char **includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error); - - const char **includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error); -#endif // WITH_CONFIG - - // Load configuration from standard input (stdim). - FILE *loadFromStdio(); - - // Load configuration from local file. - FILE *loadFromLocalFile(const std::string &u); - - std::list resolveIncludes(const std::string &name); - - void resolveEnvVars(std::string &text); - - // Resolve custom include directives. - json_t *expandIncludes(json_t *in); - - // To shell-like subsitution of environment variables in strings. - json_t *expandEnvVars(json_t *in); - - // Run a callback function for each string in the config - json_t *walkStrings(json_t *in, str_walk_fcn_t cb); - - // Get the include dirs - std::list getIncludeDirectories(FILE *f) const; - -public: - json_t *root; - - Config(); - Config(const std::string &u); - - ~Config(); - - json_t *load(std::FILE *f, bool resolveIncludes = true, - bool resolveEnvVars = true); - - json_t *load(const std::string &u, bool resolveIncludes = true, - bool resolveEnvVars = true); - - std::string const &getConfigPath() const { return configPath; } -}; - -} // namespace node -} // namespace villas diff --git a/include/villas/node/exceptions.hpp b/include/villas/node/exceptions.hpp index e84eba2a9..5990fa55c 100644 --- a/include/villas/node/exceptions.hpp +++ b/include/villas/node/exceptions.hpp @@ -10,10 +10,6 @@ #include #include -#ifdef WITH_CONFIG -#include -#endif - namespace villas { namespace node { @@ -31,21 +27,6 @@ class ParseError : public RuntimeError { text(t), file(f), line(l), column(c) {} }; -#ifdef WITH_CONFIG -class LibconfigParseError : public ParseError { - -protected: - const config_t *config; - -public: - LibconfigParseError(const config_t *c) - : ParseError(config_error_text(c), - config_error_file(c) ? config_error_file(c) : "", - config_error_line(c)), - config(c) {} -}; -#endif // WITH_CONFIG - class JanssonParseError : public ParseError { protected: diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index f4836fdfd..7f197ebfb 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -15,11 +15,10 @@ extern "C" { } #endif -#include - #include #include -#include +#include +#include #include #include #include @@ -68,7 +67,8 @@ class SuperNode { struct timespec started; // The time at which the instance has been started. - Config config; // The configuration file. + fs::path configPath; + JanssonPtr configRoot; // The configuration file. public: // Inititalize configuration object before parsing the configuration. @@ -77,7 +77,7 @@ class SuperNode { int init(); // Wrapper for parse() which loads the config first. - void parse(const std::string &name); + void parse(fs::path const &path); /* Parse super-node configuration. * @@ -138,9 +138,9 @@ class SuperNode { Web *getWeb() { return &web; } #endif - json_t *getConfig() { return config.root; } + json_t *getConfig() { return configRoot.get(); } - const std::string &getConfigPath() const { return config.getConfigPath(); } + fs::path const &getConfigPath() const { return configPath; } int getAffinity() const { return affinity; } diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e95215e5a..e0e904018 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -23,7 +23,6 @@ set(LIBRARIES set(LIB_SRC capabilities.cpp config_helper.cpp - config.cpp dumper.cpp format.cpp mapping.cpp diff --git a/lib/config.cpp b/lib/config.cpp deleted file mode 100644 index 0bab007ef..000000000 --- a/lib/config.cpp +++ /dev/null @@ -1,368 +0,0 @@ -/* Configuration file parsing. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#ifdef WITH_CONFIG -#include -#endif - -using namespace villas; -using namespace villas::node; - -Config::Config() : logger(Log::get("config")), root(nullptr) {} - -Config::Config(const std::string &u) : Config() { root = load(u); } - -Config::~Config() { json_decref(root); } - -json_t *Config::load(std::FILE *f, bool resolveInc, bool resolveEnvVars) { - json_t *root = decode(f); - - if (resolveInc) { - json_t *root_old = root; - root = expandIncludes(root); - json_decref(root_old); - } - - if (resolveEnvVars) { - json_t *root_old = root; - root = expandEnvVars(root); - json_decref(root_old); - } - - return root; -} - -json_t *Config::load(const std::string &u, bool resolveInc, - bool resolveEnvVars) { - FILE *f; - - if (u == "-") - f = loadFromStdio(); - else - f = loadFromLocalFile(u); - - json_t *root = load(f, resolveInc, resolveEnvVars); - - fclose(f); - - return root; -} - -FILE *Config::loadFromStdio() { - logger->info("Reading configuration from standard input"); - - auto *cwd = new char[PATH_MAX]; - - configPath = getcwd(cwd, PATH_MAX); - - delete[] cwd; - - return stdin; -} - -FILE *Config::loadFromLocalFile(const std::string &u) { - logger->info("Reading configuration from local file: {}", u); - - configPath = u; - FILE *f = fopen(u.c_str(), "r"); - if (!f) - throw RuntimeError("Failed to open configuration from: {}", u); - - return f; -} - -json_t *Config::decode(FILE *f) { - json_error_t err; - - // Update list of include directories - auto incDirs = getIncludeDirectories(f); - includeDirectories.insert(includeDirectories.end(), incDirs.begin(), - incDirs.end()); - - json_t *root = json_loadf(f, 0, &err); - if (root == nullptr) { -#ifdef WITH_CONFIG - // We try again to parse the config in the legacy format - root = libconfigDecode(f); -#else - throw JanssonParseError(err); -#endif // WITH_CONFIG - } - - return root; -} - -std::list Config::getIncludeDirectories(FILE *f) const { - int ret, fd; - char buf[PATH_MAX]; - char *dir; - - std::list dirs; - - // Adding directory of base configuration file - fd = fileno(f); - if (fd < 0) - throw SystemError("Failed to get file descriptor"); - - auto path = fmt::format("/proc/self/fd/{}", fd); - - ret = readlink(path.c_str(), buf, sizeof(buf)); - if (ret > 0) { - buf[ret] = 0; - if (isLocalFile(buf)) { - dir = dirname(buf); - dirs.push_back(dir); - } - } - - // Adding current working directory - dir = getcwd(buf, sizeof(buf)); - if (dir != nullptr) - dirs.push_back(dir); - - return dirs; -} - -std::list Config::resolveIncludes(const std::string &n) { - glob_t gb; - int ret, flags = 0; - - memset(&gb, 0, sizeof(gb)); - - auto name = n; - resolveEnvVars(name); - - if (name.size() >= 1 && name[0] == '/') { // absolute path - ret = glob(name.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) - gb.gl_pathc = 0; - } else { // relative path - for (auto &dir : includeDirectories) { - auto pattern = fmt::format("{}/{}", dir, name.c_str()); - - ret = glob(pattern.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) { - gb.gl_pathc = 0; - - goto out; - } - - flags |= GLOB_APPEND; - } - } - -out: - std::list files; - for (unsigned i = 0; i < gb.gl_pathc; i++) - files.push_back(gb.gl_pathv[i]); - - globfree(&gb); - - return files; -} - -void Config::resolveEnvVars(std::string &text) { - static const std::regex env_re{R"--(\$\{([^}]+)\})--"}; - - std::smatch match; - while (std::regex_search(text, match, env_re)) { - auto const from = match[0]; - auto const var_name = match[1].str(); - char *var_value = std::getenv(var_name.c_str()); - if (!var_value) - throw RuntimeError("Unresolved environment variable: {}", var_name); - - text.replace(from.first - text.begin(), from.second - from.first, - var_value); - - logger->debug("Replace env var {} in \"{}\" with value \"{}\"", var_name, - text, var_value); - } -} - -#ifdef WITH_CONFIG -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) -const char **Config::includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - void *ctx = config_get_hook(cfg); - - return reinterpret_cast(ctx)->includeFunc(cfg, include_dir, path, - error); -} - -const char **Config::includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - auto paths = resolveIncludes(path); - - unsigned i = 0; - auto files = (const char **)malloc(sizeof(char **) * (paths.size() + 1)); - - for (auto &path : paths) - files[i++] = strdup(path.c_str()); - - files[i] = NULL; - - return files; -} -#endif - -json_t *Config::libconfigDecode(FILE *f) { - int ret; - - config_t cfg; - config_setting_t *cfg_root; - config_init(&cfg); - config_set_auto_convert(&cfg, 1); - - // Setup libconfig include path -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) - config_set_hook(&cfg, this); - - config_set_include_func(&cfg, includeFuncStub); -#else - if (includeDirectories.size() > 0) { - logger->info("Setting include dir to: {}", includeDirectories.front()); - - config_set_include_dir(&cfg, includeDirectories.front().c_str()); - - if (includeDirectories.size() > 1) { - logger->warn( - "Ignoring all but the first include directories for libconfig"); - logger->warn( - " libconfig does not support more than a single include dir!"); - } - } -#endif - - // Rewind before re-reading - rewind(f); - - ret = config_read(&cfg, f); - if (ret != CONFIG_TRUE) - throw LibconfigParseError(&cfg); - - cfg_root = config_root_setting(&cfg); - - json_t *root = config_to_json(cfg_root); - if (!root) - throw RuntimeError("Failed to convert JSON to configuration file"); - - config_destroy(&cfg); - - return root; -} -#endif // WITH_CONFIG - -json_t *Config::walkStrings(json_t *root, str_walk_fcn_t cb) { - const char *key; - size_t index; - json_t *val, *new_val, *new_root; - - switch (json_typeof(root)) { - case JSON_STRING: - return cb(root); - - case JSON_OBJECT: - new_root = json_object(); - - json_object_foreach (root, key, val) { - new_val = walkStrings(val, cb); - - json_object_set_new(new_root, key, new_val); - } - - return new_root; - - case JSON_ARRAY: - new_root = json_array(); - - json_array_foreach (root, index, val) { - new_val = walkStrings(val, cb); - - json_array_append_new(new_root, new_val); - } - - return new_root; - - default: - return json_incref(root); - }; -} - -json_t *Config::expandEnvVars(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - std::string text = json_string_value(str); - - resolveEnvVars(text); - - return json_string(text.c_str()); - }); -} - -json_t *Config::expandIncludes(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - int ret; - std::string text = json_string_value(str); - static const std::string kw = "@include "; - - auto res = std::mismatch(kw.begin(), kw.end(), text.begin()); - if (res.first != kw.end()) - return json_incref(str); - else { - std::string pattern = text.substr(kw.size()); - - resolveEnvVars(pattern); - - json_t *incl = nullptr; - - for (auto &path : resolveIncludes(pattern)) { - json_t *other = load(path); - if (!other) - throw ConfigError(str, "include", - "Failed to include config file from {}", path); - - if (!incl) - incl = other; - else if (json_is_object(incl) && json_is_object(other)) { - ret = json_object_update_recursive(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } else if (json_is_array(incl) && json_is_array(other)) { - ret = json_array_extend(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } - - logger->debug("Included config from: {}", path); - } - - return incl; - } - }); -} diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp index 4d8ccc3f3..e4af36d3b 100644 --- a/lib/nodes/fpga.cpp +++ b/lib/nodes/fpga.cpp @@ -381,8 +381,7 @@ int FpgaNodeFactory::start(SuperNode *sn) { } if (cards.empty()) { - auto searchPath = - sn->getConfigPath().substr(0, sn->getConfigPath().rfind("/")); + auto searchPath = sn->getConfigPath(); createCards(sn->getConfig(), cards, searchPath, vfioContainer); } diff --git a/lib/super_node.cpp b/lib/super_node.cpp index 110d194c3..6dc94eeb7 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -7,7 +7,8 @@ #include #include -#include + +#include #include #include @@ -22,6 +23,8 @@ #include #include +#include "villas/json.hpp" + #ifdef WITH_NETEM #include #endif @@ -60,10 +63,18 @@ SuperNode::SuperNode() logger = Log::get("super_node"); } -void SuperNode::parse(const std::string &u) { - config.root = config.load(u); +void SuperNode::parse(fs::path const &path) { + configPath = path; + + load_config_file(path, + { + .allow_libconfig = true, + .allow_environment = true, + .allow_include = true, + }) + .get_to(configRoot); - parse(config.root); + parse(configRoot.get()); } void SuperNode::parse(json_t *root) { diff --git a/packaging/deps.sh b/packaging/deps.sh index 35e0f840c..d3270a27a 100644 --- a/packaging/deps.sh +++ b/packaging/deps.sh @@ -602,6 +602,19 @@ if ! cmake --find-package -DNAME=ghc_filesystem -DCOMPILER_ID=GNU -DLANGUAGE=CXX popd fi +# Build and install nlohmann_json +if ! pkg-config "nlohmann_json" && + should_build "nlohman_json" "for configuration parsing"; then + git clone --branch v3.12.0 https://github.com/nlohmann/json.git json + mkdir -p json/build + pushd json/build + cmake ${CMAKE_OPTS} .. + cmake --build . \ + --target install \ + --parallel ${PARALLEL} + popd +fi + popd >/dev/null # Update linker cache diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index 631e16f86..8a35b80a4 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -52,7 +52,8 @@ RUN apt-get update && \ libssl-dev \ libusb-1.0-0-dev \ libzmq3-dev \ - uuid-dev + uuid-dev \ + nlohmann-json3-dev # Install unpackaged dependencies from source ADD packaging/patches /deps/patches diff --git a/packaging/docker/Dockerfile.debian-multiarch b/packaging/docker/Dockerfile.debian-multiarch index 6007a9efd..b34711ca5 100644 --- a/packaging/docker/Dockerfile.debian-multiarch +++ b/packaging/docker/Dockerfile.debian-multiarch @@ -58,7 +58,8 @@ RUN apt-get update && \ libssl-dev:${ARCH} \ libusb-1.0-0-dev:${ARCH} \ libzmq3-dev:${ARCH} \ - uuid-dev:${ARCH} + uuid-dev:${ARCH} \ + nlohmann-json3-dev:${ARCH} ADD cmake/toolchains/debian-${ARCH}.cmake / diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index c78a8fca0..c1e0ba250 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -65,7 +65,8 @@ RUN dnf -y install \ protobuf-c-devel \ protobuf-devel \ spdlog-devel \ - zeromq-devel + zeromq-devel \ + json-devel # Install unpackaged dependencies from source # TODO: We currently need to build with GCC 14 to get OpenDSSC working diff --git a/packaging/docker/Dockerfile.fedora-minimal b/packaging/docker/Dockerfile.fedora-minimal index 749bbedc6..c9d779371 100644 --- a/packaging/docker/Dockerfile.fedora-minimal +++ b/packaging/docker/Dockerfile.fedora-minimal @@ -24,7 +24,8 @@ RUN dnf -y install \ jansson-devel \ spdlog-devel \ fmt-devel \ - libwebsockets-devel + libwebsockets-devel \ + json-devel ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index 384bf6d68..01a34338f 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -53,7 +53,8 @@ RUN dnf -y install \ nanomsg-devel \ libnice-devel \ libre-devel \ - libwebsockets-devel + libwebsockets-devel \ + json-devel # Install unpackaged dependencies from source ADD packaging/patches /deps/patches diff --git a/packaging/docker/Dockerfile.rocky9 b/packaging/docker/Dockerfile.rocky9 index f7dfffd40..b0d762ebf 100644 --- a/packaging/docker/Dockerfile.rocky9 +++ b/packaging/docker/Dockerfile.rocky9 @@ -48,7 +48,8 @@ RUN dnf -y install \ lua-devel \ hiredis-devel \ libnice-devel \ - libmodbus-devel + libmodbus-devel \ + json-devel # Install unpackaged dependencies from source ADD packaging/patches /deps/patches diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index 76c21e2a7..bc0b86181 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -64,7 +64,8 @@ RUN apt-get update && \ libusb-1.0-0-dev \ libwebsockets-dev \ libzmq3-dev \ - uuid-dev + uuid-dev \ + nlohmann-json3-dev # Install unpackaged dependencies from source ADD packaging/patches /deps/patches diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix index 2826816c0..e9874d910 100644 --- a/packaging/nix/villas.nix +++ b/packaging/nix/villas.nix @@ -1,9 +1,13 @@ # SPDX-FileCopyrightText: 2023 OPAL-RT Germany GmbH # SPDX-License-Identifier: Apache-2.0 { + lib, + stdenv, + makeWrapper, # General configuration src, version, + system, withGpl ? true, withAllExtras ? false, withAllFormats ? false, @@ -41,23 +45,25 @@ bash, cmake, coreutils, + curl, + gnugrep, graphviz, + jansson, jq, - lib, - makeWrapper, + libuuid, + libwebsockets, + nlohmann_json, + openssl, pkg-config, gcc14Stdenv, - system, + spdlog, # Optional dependencies boxfort, comedilib, criterion, - curl, czmq, cyrus_sasl, ethercat, - gnugrep, - jansson, lib60870, libconfig, libdatachannel, @@ -70,14 +76,11 @@ libsodium, libuldaq, libusb1, - libuuid, - libwebsockets, libxml2, lua, mosquitto, nanomsg, opendssc, - openssl, orchestra, pcre2, pkgsBuildBuild, @@ -89,7 +92,6 @@ rdkafka, rdma-core, redis-plus-plus, - spdlog, linuxHeaders, }: @@ -193,8 +195,9 @@ gcc14Stdenv.mkDerivation { ]; propagatedBuildInputs = [ - libuuid jansson + libuuid + nlohmann_json ] ++ lib.optionals withFormatProtobuf [ protobuf diff --git a/tests/unit/config.cpp b/tests/unit/config.cpp index 0ecd3a2fe..a04315320 100644 --- a/tests/unit/config.cpp +++ b/tests/unit/config.cpp @@ -9,68 +9,99 @@ #include #include +#include +#include -#include +#include +#include +#include #include -using namespace villas::node; +using namespace std::string_view_literals; +using FileDeleter = decltype([](std::FILE *f) { std::fclose(f); }); +using FilePtr = std::unique_ptr; + +constexpr auto fileNameTemplate = "villas.unit-test.XXXXXX.conf"sv; +constexpr auto fileNameSuffix = ".conf"sv; // cppcheck-suppress syntaxError Test(config, env) { - const char *cfg_f = "test = \"${MY_ENV_VAR}\"\n"; - - std::FILE *f = std::tmpfile(); - std::fputs(cfg_f, f); - std::rewind(f); - - auto c = Config(); - - char env[] = "MY_ENV_VAR=mobydick"; - putenv(env); - - auto *r = c.load(f); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "test"); - cr_assert_not_null(j); - - cr_assert(json_is_string(j)); - cr_assert_str_eq("mobydick", json_string_value(j)); + auto config_string = R"libconfig( + test = "${MY_ENV_VAR}" + )libconfig"; + + auto config_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto config_fd = + ::mkstemps(config_path_template.data(), fileNameSuffix.length()); + auto config_file = FilePtr(::fdopen(config_fd, "w")); + auto config_path = fs::path(config_path_template); + std::fputs(config_string, config_file.get()); + config_file.reset(); + + ::setenv("MY_ENV_VAR", "mobydick", true); + auto config = villas::load_config_file(config_path, + { + .allow_libconfig = true, + .allow_environment = true, + }) + .get(); + + auto *root = config.get(); + cr_assert_not_null(root); + + auto *string = json_object_get(root, "test"); + cr_assert_not_null(string); + cr_assert(json_is_string(string)); + cr_assert_str_eq("mobydick", json_string_value(string)); } Test(config, include) { - const char *cfg_f2 = "magic = 1234\n"; - - char f2_fn_tpl[] = "/tmp/villas.unit-test.XXXXXX"; - int f2_fd = mkstemp(f2_fn_tpl); - - std::FILE *f2 = fdopen(f2_fd, "w"); - std::fputs(cfg_f2, f2); - std::rewind(f2); - - auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn_tpl); - - std::FILE *f1 = std::tmpfile(); - std::fputs(cfg_f1.c_str(), f1); - std::rewind(f1); - - auto env = fmt::format("{}", f2_fn_tpl); - setenv("INCLUDE_FILE", env.c_str(), true); - - auto c = Config(); - - auto *r = c.load(f1); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "subval"); - cr_assert_not_null(j); - - auto *j2 = json_object_get(j, "magic"); - cr_assert_not_null(j2); - - cr_assert(json_is_integer(j2)); - cr_assert_eq(json_number_value(j2), 1234); - - std::fclose(f2); - std::remove(f2_fn_tpl); + auto incString = R"libconfig( + magic = 1234 + )libconfig"; + + auto inc_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto inc_fd = ::mkstemps(inc_path_template.data(), fileNameSuffix.length()); + cr_assert(inc_fd >= 0); + auto inc_file = FilePtr(::fdopen(inc_fd, "w")); + cr_assert_not_null(inc_file); + auto inc_path = fs::path(inc_path_template); + std::fputs(incString, inc_file.get()); + inc_file.reset(); + + auto config_string = fmt::format(R"libconfig( + subval = {{ + @include "{}" + }} + )libconfig", + inc_path.string()); + auto config_path_template = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto config_fd = + ::mkstemps(config_path_template.data(), fileNameSuffix.length()); + cr_assert(config_fd >= 0); + auto config_file = FilePtr(::fdopen(config_fd, "w")); + cr_assert_not_null(config_file); + auto config_path = fs::path(config_path_template); + std::fputs(config_string.c_str(), config_file.get()); + config_file.reset(); + + auto config = villas::load_config_file(config_path, + { + .allow_libconfig = true, + .allow_include = true, + }) + .get(); + auto *root = config.get(); + cr_assert_not_null(root); + + auto *subval = json_object_get(root, "subval"); + cr_assert_not_null(subval); + + auto *magic = json_object_get(subval, "magic"); + cr_assert_not_null(magic); + cr_assert(json_is_integer(magic)); + cr_assert_eq(json_number_value(magic), 1234); } diff --git a/tests/unit/config_json.cpp b/tests/unit/config_json.cpp index 268385d82..b62d6cb65 100644 --- a/tests/unit/config_json.cpp +++ b/tests/unit/config_json.cpp @@ -9,10 +9,10 @@ #ifdef WITH_CONFIG -#include #include #include +#include #include using namespace villas::node;