From 7d9e3fbefa1d2e9f7166eaad55ca11dc09d1ff8a Mon Sep 17 00:00:00 2001 From: arnavdham Date: Fri, 31 Jul 2026 22:46:31 +0530 Subject: [PATCH 1/8] dynamic loading --- cmake/podioTest.cmake | 1 + .../podio/utilities/ArrowConverterRegistry.h | 19 +++++ src/ArrowConverterRegistry.cc | 69 +++++++++++++++++++ tests/unittests/CMakeLists.txt | 12 +--- 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/cmake/podioTest.cmake b/cmake/podioTest.cmake index 5ab39a3c0..1b99eafca 100644 --- a/cmake/podioTest.cmake +++ b/cmake/podioTest.cmake @@ -45,6 +45,7 @@ function(PODIO_SET_TEST_ENV test) LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/src:$:$<$:$>:$ENV{LD_LIBRARY_PATH} PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH} PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests + PODIO_ARROW_PATH=${PROJECT_BINARY_DIR}/tests ROOT_INCLUDE_PATH=${PROJECT_SOURCE_DIR}/tests:${PROJECT_SOURCE_DIR}/include:$ENV{ROOT_INCLUDE_PATH} SKIP_SIO_TESTS=$> IO_HANDLERS=${IO_HANDLERS} diff --git a/include/podio/utilities/ArrowConverterRegistry.h b/include/podio/utilities/ArrowConverterRegistry.h index 6d3200314..fc65fa765 100644 --- a/include/podio/utilities/ArrowConverterRegistry.h +++ b/include/podio/utilities/ArrowConverterRegistry.h @@ -69,6 +69,25 @@ class ArrowConverterRegistry { std::map m_readerRegistry; }; +class ArrowConverterLibraryLoader { +private: + ArrowConverterLibraryLoader(); + + enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 }; + + LoadStatus loadLib(const std::string& libname, const std::string& directory); + + static std::vector> getLibNames(); + + std::map m_loadedLibs{}; + +public: + static ArrowConverterLibraryLoader& instance() { + static ArrowConverterLibraryLoader me; + return me; + } +}; + } // namespace podio #endif // PODIO_ARROWCONVERTERREGISTRY_H diff --git a/src/ArrowConverterRegistry.cc b/src/ArrowConverterRegistry.cc index 7a18da609..8f25485d3 100644 --- a/src/ArrowConverterRegistry.cc +++ b/src/ArrowConverterRegistry.cc @@ -1,4 +1,9 @@ #include "podio/utilities/ArrowConverterRegistry.h" +#include +#include +#include +#include +#include namespace podio { @@ -16,6 +21,7 @@ void ArrowConverterRegistry::registerConverter(const std::string& typeName, Crea } ArrowConverterRegistry::CreatorFunc ArrowConverterRegistry::getConverter(const std::string& typeName) const { + ArrowConverterLibraryLoader::instance(); auto it = m_registry.find(typeName); if (it != m_registry.end()) { return it->second; @@ -28,6 +34,7 @@ void ArrowConverterRegistry::registerReader(const std::string& typeName, BufferR } ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const std::string& typeName) const { + ArrowConverterLibraryLoader::instance(); auto it = m_readerRegistry.find(typeName); if (it != m_readerRegistry.end()) { return it->second; @@ -35,4 +42,66 @@ ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const return nullptr; } +ArrowConverterLibraryLoader::ArrowConverterLibraryLoader() { + for (const auto& [lib, dir] : getLibNames()) { + const auto status = loadLib(lib, dir); + switch (status) { + case LoadStatus::Success: + break; + case LoadStatus::AlreadyLoaded: + break; + case LoadStatus::Error: + std::cerr << "ERROR while loading Arrow library \'" << lib << "\' (from " << dir << ")" << std::endl; + break; + } + } +} + +ArrowConverterLibraryLoader::LoadStatus ArrowConverterLibraryLoader::loadLib(const std::string& libname, + const std::string& directory) { + if (m_loadedLibs.find(libname) != m_loadedLibs.end()) { + return LoadStatus::AlreadyLoaded; + } + void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL); + if (libhandle) { + m_loadedLibs.insert({libname, libhandle}); + return LoadStatus::Success; + } + + return LoadStatus::Error; +} + +std::vector> ArrowConverterLibraryLoader::getLibNames() { + namespace fs = std::filesystem; + std::vector> libs; + + const auto ldLibPath = []() { + auto pathVar = std::getenv("PODIO_ARROW_PATH"); + if (!pathVar) { + pathVar = std::getenv("LD_LIBRARY_PATH"); + } + return pathVar; + }(); + if (!ldLibPath) { + return libs; + } + + std::string dir; + std::istringstream stream(ldLibPath); + while (std::getline(stream, dir, ':')) { + if (not fs::exists(dir)) { + continue; + } + + for (auto& lib : fs::directory_iterator(dir)) { + const auto filename = lib.path().filename().string(); + if (filename.find("Arrow") != std::string::npos && filename.find("libarrow") == std::string::npos) { + libs.emplace_back(std::move(filename), dir); + } + } + } + + return libs; +} + } // namespace podio diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index a33fdfee7..9f16609c6 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -53,21 +53,11 @@ endif() if (ENABLE_ARROW) target_sources(unittest_podio PRIVATE test_arrow_converter.cpp) target_link_libraries(unittest_podio PRIVATE podio::podioArrow ${PODIO_ARROW_TARGET}) - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(unittest_podio PRIVATE - -Wl,--push-state,--no-as-needed TestDataModelArrow ExtensionDataModelArrow InterfaceExtensionDataModelArrow - -Wl,--pop-state ) - else() - target_link_libraries(unittest_podio PRIVATE - TestDataModelArrow - ExtensionDataModelArrow - InterfaceExtensionDataModelArrow - ) - endif() target_compile_definitions(unittest_podio PRIVATE PODIO_ENABLE_ARROW=1) endif() @@ -109,6 +99,8 @@ else() ENVIRONMENT PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests ENVIRONMENT + PODIO_ARROW_PATH=${PROJECT_BINARY_DIR}/tests + ENVIRONMENT ROOT_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${PROJECT_BINARY_DIR}/src:${PROJECT_BINARY_DIR}/tests:$:$<$:$>:$ENV{LD_LIBRARY_PATH} From 79ea01c5219ddeac146d59d04e1cc438efb18e7e Mon Sep 17 00:00:00 2001 From: arnavdham Date: Fri, 31 Jul 2026 23:44:03 +0530 Subject: [PATCH 2/8] In type registry also --- src/ArrowTypeRegistry.cc | 2 ++ tests/unittests/CMakeLists.txt | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ArrowTypeRegistry.cc b/src/ArrowTypeRegistry.cc index f9c6d898b..3d903a06a 100644 --- a/src/ArrowTypeRegistry.cc +++ b/src/ArrowTypeRegistry.cc @@ -1,4 +1,5 @@ #include "podio/utilities/ArrowTypeRegistry.h" +#include "podio/utilities/ArrowConverterRegistry.h" namespace podio { @@ -16,6 +17,7 @@ void ArrowTypeRegistry::registerType(const std::string& typeName, std::shared_pt } std::shared_ptr ArrowTypeRegistry::getType(const std::string& typeName) const { + ArrowConverterLibraryLoader::instance(); auto it = m_registry.find(typeName); if (it != m_registry.end()) { return it->second; diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 9f16609c6..984168a4b 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -53,11 +53,11 @@ endif() if (ENABLE_ARROW) target_sources(unittest_podio PRIVATE test_arrow_converter.cpp) target_link_libraries(unittest_podio PRIVATE podio::podioArrow ${PODIO_ARROW_TARGET}) - target_link_libraries(unittest_podio PRIVATE - TestDataModelArrow - ExtensionDataModelArrow - InterfaceExtensionDataModelArrow - ) + add_dependencies(unittest_podio + TestDataModelArrow + ExtensionDataModelArrow + InterfaceExtensionDataModelArrow + ) target_compile_definitions(unittest_podio PRIVATE PODIO_ENABLE_ARROW=1) endif() From b28d7a10671ac198435ec39b050e99d5121cefeb Mon Sep 17 00:00:00 2001 From: arnavdham Date: Tue, 4 Aug 2026 20:22:52 +0530 Subject: [PATCH 3/8] comments for clarification --- include/podio/utilities/ArrowConverterRegistry.h | 4 ++++ include/podio/utilities/ArrowTypeRegistry.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/include/podio/utilities/ArrowConverterRegistry.h b/include/podio/utilities/ArrowConverterRegistry.h index fc65fa765..bdce05e04 100644 --- a/include/podio/utilities/ArrowConverterRegistry.h +++ b/include/podio/utilities/ArrowConverterRegistry.h @@ -20,6 +20,10 @@ class CollectionBase; /** * @brief Global singleton registry mapping PODIO type name strings to their * corresponding Apache Arrow array converter callbacks. + * + * Registration is expected to happen during shared library loading/startup, + * before worker threads query the registry. After registration the registry is + * read-only and can be queried concurrently. */ class ArrowConverterRegistry { public: diff --git a/include/podio/utilities/ArrowTypeRegistry.h b/include/podio/utilities/ArrowTypeRegistry.h index 0cdaf1ea1..46be8d216 100644 --- a/include/podio/utilities/ArrowTypeRegistry.h +++ b/include/podio/utilities/ArrowTypeRegistry.h @@ -15,6 +15,10 @@ namespace podio { /** * @brief Global singleton registry mapping PODIO type name strings to their * corresponding Apache Arrow DataTypes. + * + * Registration is expected to happen during shared library loading/startup, + * before worker threads query the registry. After registration the registry is + * read-only and can be queried concurrently. */ class ArrowTypeRegistry { public: From ba66727f0b1311dfd1517edfd3b32a58ae597874 Mon Sep 17 00:00:00 2001 From: arnavdham Date: Sat, 8 Aug 2026 11:27:59 +0530 Subject: [PATCH 4/8] Comments resolution. --- include/podio/utilities/ArrowConverterRegistry.h | 8 +++++--- include/podio/utilities/ArrowTypeRegistry.h | 8 +++++--- src/ArrowConverterRegistry.cc | 11 ++++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/include/podio/utilities/ArrowConverterRegistry.h b/include/podio/utilities/ArrowConverterRegistry.h index bdce05e04..91f591473 100644 --- a/include/podio/utilities/ArrowConverterRegistry.h +++ b/include/podio/utilities/ArrowConverterRegistry.h @@ -21,9 +21,11 @@ class CollectionBase; * @brief Global singleton registry mapping PODIO type name strings to their * corresponding Apache Arrow array converter callbacks. * - * Registration is expected to happen during shared library loading/startup, - * before worker threads query the registry. After registration the registry is - * read-only and can be queried concurrently. + * Registration happens lazily on the first call to getConverter or getReader, + * when the necessary datamodel-specific Arrow converter libraries (e.g., + * libpodioArrow.so, libedm4hepArrow.so) are loaded. It is expected that this + * happens before worker threads query the registry. Once populated, the registry + * is read-only and can be safely accessed from multiple threads concurrently. */ class ArrowConverterRegistry { public: diff --git a/include/podio/utilities/ArrowTypeRegistry.h b/include/podio/utilities/ArrowTypeRegistry.h index 46be8d216..a948cde9b 100644 --- a/include/podio/utilities/ArrowTypeRegistry.h +++ b/include/podio/utilities/ArrowTypeRegistry.h @@ -16,9 +16,11 @@ namespace podio { * @brief Global singleton registry mapping PODIO type name strings to their * corresponding Apache Arrow DataTypes. * - * Registration is expected to happen during shared library loading/startup, - * before worker threads query the registry. After registration the registry is - * read-only and can be queried concurrently. + * Registration happens lazily on the first call to getType, when the necessary + * datamodel-specific Arrow converter libraries (e.g., libpodioArrow.so, + * libedm4hepArrow.so) are loaded. It is expected that this happens before + * worker threads query the registry. Once populated, the registry is read-only + * and can be safely accessed from multiple threads concurrently. */ class ArrowTypeRegistry { public: diff --git a/src/ArrowConverterRegistry.cc b/src/ArrowConverterRegistry.cc index 8f25485d3..b4dcef5ce 100644 --- a/src/ArrowConverterRegistry.cc +++ b/src/ArrowConverterRegistry.cc @@ -47,13 +47,18 @@ ArrowConverterLibraryLoader::ArrowConverterLibraryLoader() { const auto status = loadLib(lib, dir); switch (status) { case LoadStatus::Success: + std::cerr << "Loaded Arrow library \'" << lib << "\' (from " << dir << ")" << std::endl; break; case LoadStatus::AlreadyLoaded: + std::cerr << "Arrow library \'" << lib << "\' already loaded. Not loading again from " << dir << std::endl; break; - case LoadStatus::Error: - std::cerr << "ERROR while loading Arrow library \'" << lib << "\' (from " << dir << ")" << std::endl; + case LoadStatus::Error: { + const char* err = dlerror(); + std::cerr << "ERROR while loading Arrow library \'" << lib << "\' (from " << dir + << "): " << (err ? err : "Unknown error") << std::endl; break; } + } } } @@ -95,7 +100,7 @@ std::vector> ArrowConverterLibraryLoader::g for (auto& lib : fs::directory_iterator(dir)) { const auto filename = lib.path().filename().string(); - if (filename.find("Arrow") != std::string::npos && filename.find("libarrow") == std::string::npos) { + if (filename.find("Arrow") != std::string::npos) { libs.emplace_back(std::move(filename), dir); } } From d4de8aba8369e17d9ee24b3a723f56122fdbf72b Mon Sep 17 00:00:00 2001 From: arnavdham Date: Mon, 10 Aug 2026 21:12:19 +0530 Subject: [PATCH 5/8] removing libpodioArrow.so --- include/podio/utilities/ArrowConverterRegistry.h | 2 +- include/podio/utilities/ArrowTypeRegistry.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/podio/utilities/ArrowConverterRegistry.h b/include/podio/utilities/ArrowConverterRegistry.h index 91f591473..e313f4fe5 100644 --- a/include/podio/utilities/ArrowConverterRegistry.h +++ b/include/podio/utilities/ArrowConverterRegistry.h @@ -23,7 +23,7 @@ class CollectionBase; * * Registration happens lazily on the first call to getConverter or getReader, * when the necessary datamodel-specific Arrow converter libraries (e.g., - * libpodioArrow.so, libedm4hepArrow.so) are loaded. It is expected that this + * libedm4hepArrow.so, libTestDataModelArrow.so) are loaded. It is expected that this * happens before worker threads query the registry. Once populated, the registry * is read-only and can be safely accessed from multiple threads concurrently. */ diff --git a/include/podio/utilities/ArrowTypeRegistry.h b/include/podio/utilities/ArrowTypeRegistry.h index a948cde9b..ffde424a9 100644 --- a/include/podio/utilities/ArrowTypeRegistry.h +++ b/include/podio/utilities/ArrowTypeRegistry.h @@ -17,8 +17,8 @@ namespace podio { * corresponding Apache Arrow DataTypes. * * Registration happens lazily on the first call to getType, when the necessary - * datamodel-specific Arrow converter libraries (e.g., libpodioArrow.so, - * libedm4hepArrow.so) are loaded. It is expected that this happens before + * datamodel-specific Arrow converter libraries (e.g., libedm4hepArrow.so, + * libTestDataModelArrow.so) are loaded. It is expected that this happens before * worker threads query the registry. Once populated, the registry is read-only * and can be safely accessed from multiple threads concurrently. */ From 27c31b593119afe2d92daea83379708f2541fe10 Mon Sep 17 00:00:00 2001 From: arnavdham Date: Mon, 10 Aug 2026 21:44:44 +0530 Subject: [PATCH 6/8] generic library loader --- include/podio/SIOBlock.h | 23 +---- .../podio/utilities/ArrowConverterRegistry.h | 19 +--- include/podio/utilities/LibraryLoader.h | 32 +++++++ src/ArrowConverterRegistry.cc | 72 +-------------- src/ArrowTypeRegistry.cc | 2 +- src/CMakeLists.txt | 1 + src/LibraryLoader.cc | 91 +++++++++++++++++++ src/SIOBlock.cc | 73 +-------------- src/SIOLegacyReader.cc | 2 +- src/SIOReader.cc | 2 +- src/SIOWriter.cc | 2 +- 11 files changed, 139 insertions(+), 180 deletions(-) create mode 100644 include/podio/utilities/LibraryLoader.h create mode 100644 src/LibraryLoader.cc diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index eb41d90a8..c24f58bc2 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -255,28 +255,7 @@ class SIOBlockFactory { } }; -class SIOBlockLibraryLoader { -private: - SIOBlockLibraryLoader(); - - /// Status code for loading shared SIOBlocks libraries - enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 }; - - /// Load a library with the given name via dlopen - LoadStatus loadLib(const std::string& libname, const std::string& directory); - - /// Get all files that are found on LD_LIBRARY_PATH and that have "SioBlocks" - /// in their name together with the directory they are in - static std::vector> getLibNames(); - - std::map _loadedLibs{}; - -public: - static SIOBlockLibraryLoader& instance() { - static SIOBlockLibraryLoader instance; - return instance; - } -}; +void loadSIOBlocksLibraries(); namespace sio_helpers { /// marker for showing that a TOC has been stored in the file diff --git a/include/podio/utilities/ArrowConverterRegistry.h b/include/podio/utilities/ArrowConverterRegistry.h index e313f4fe5..92904506a 100644 --- a/include/podio/utilities/ArrowConverterRegistry.h +++ b/include/podio/utilities/ArrowConverterRegistry.h @@ -75,24 +75,7 @@ class ArrowConverterRegistry { std::map m_readerRegistry; }; -class ArrowConverterLibraryLoader { -private: - ArrowConverterLibraryLoader(); - - enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 }; - - LoadStatus loadLib(const std::string& libname, const std::string& directory); - - static std::vector> getLibNames(); - - std::map m_loadedLibs{}; - -public: - static ArrowConverterLibraryLoader& instance() { - static ArrowConverterLibraryLoader me; - return me; - } -}; +void loadArrowLibraries(); } // namespace podio diff --git a/include/podio/utilities/LibraryLoader.h b/include/podio/utilities/LibraryLoader.h new file mode 100644 index 000000000..bdfa7a5eb --- /dev/null +++ b/include/podio/utilities/LibraryLoader.h @@ -0,0 +1,32 @@ +#ifndef PODIO_UTILITIES_LIBRARYLOADER_H +#define PODIO_UTILITIES_LIBRARYLOADER_H + +#include +#include +#include +#include + +namespace podio { +namespace utilities { + + class LibraryLoader { + public: + enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 }; + + LibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator); + ~LibraryLoader() = default; + + private: + LoadStatus loadLib(const std::string& libname, const std::string& directory); + std::vector> getLibNames() const; + + std::string m_envVarName; + std::string m_libraryPattern; + std::string m_logDesignator; + std::map m_loadedLibs{}; + }; + +} // namespace utilities +} // namespace podio + +#endif // PODIO_UTILITIES_LIBRARYLOADER_H diff --git a/src/ArrowConverterRegistry.cc b/src/ArrowConverterRegistry.cc index b4dcef5ce..e3dc7b69f 100644 --- a/src/ArrowConverterRegistry.cc +++ b/src/ArrowConverterRegistry.cc @@ -1,4 +1,5 @@ #include "podio/utilities/ArrowConverterRegistry.h" +#include "podio/utilities/LibraryLoader.h" #include #include #include @@ -21,7 +22,7 @@ void ArrowConverterRegistry::registerConverter(const std::string& typeName, Crea } ArrowConverterRegistry::CreatorFunc ArrowConverterRegistry::getConverter(const std::string& typeName) const { - ArrowConverterLibraryLoader::instance(); + loadArrowLibraries(); auto it = m_registry.find(typeName); if (it != m_registry.end()) { return it->second; @@ -34,7 +35,7 @@ void ArrowConverterRegistry::registerReader(const std::string& typeName, BufferR } ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const std::string& typeName) const { - ArrowConverterLibraryLoader::instance(); + loadArrowLibraries(); auto it = m_readerRegistry.find(typeName); if (it != m_readerRegistry.end()) { return it->second; @@ -42,71 +43,8 @@ ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const return nullptr; } -ArrowConverterLibraryLoader::ArrowConverterLibraryLoader() { - for (const auto& [lib, dir] : getLibNames()) { - const auto status = loadLib(lib, dir); - switch (status) { - case LoadStatus::Success: - std::cerr << "Loaded Arrow library \'" << lib << "\' (from " << dir << ")" << std::endl; - break; - case LoadStatus::AlreadyLoaded: - std::cerr << "Arrow library \'" << lib << "\' already loaded. Not loading again from " << dir << std::endl; - break; - case LoadStatus::Error: { - const char* err = dlerror(); - std::cerr << "ERROR while loading Arrow library \'" << lib << "\' (from " << dir - << "): " << (err ? err : "Unknown error") << std::endl; - break; - } - } - } -} - -ArrowConverterLibraryLoader::LoadStatus ArrowConverterLibraryLoader::loadLib(const std::string& libname, - const std::string& directory) { - if (m_loadedLibs.find(libname) != m_loadedLibs.end()) { - return LoadStatus::AlreadyLoaded; - } - void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL); - if (libhandle) { - m_loadedLibs.insert({libname, libhandle}); - return LoadStatus::Success; - } - - return LoadStatus::Error; -} - -std::vector> ArrowConverterLibraryLoader::getLibNames() { - namespace fs = std::filesystem; - std::vector> libs; - - const auto ldLibPath = []() { - auto pathVar = std::getenv("PODIO_ARROW_PATH"); - if (!pathVar) { - pathVar = std::getenv("LD_LIBRARY_PATH"); - } - return pathVar; - }(); - if (!ldLibPath) { - return libs; - } - - std::string dir; - std::istringstream stream(ldLibPath); - while (std::getline(stream, dir, ':')) { - if (not fs::exists(dir)) { - continue; - } - - for (auto& lib : fs::directory_iterator(dir)) { - const auto filename = lib.path().filename().string(); - if (filename.find("Arrow") != std::string::npos) { - libs.emplace_back(std::move(filename), dir); - } - } - } - - return libs; +void loadArrowLibraries() { + static podio::utilities::LibraryLoader me("PODIO_ARROW_PATH", "Arrow", "Arrow"); } } // namespace podio diff --git a/src/ArrowTypeRegistry.cc b/src/ArrowTypeRegistry.cc index 3d903a06a..b7becec2e 100644 --- a/src/ArrowTypeRegistry.cc +++ b/src/ArrowTypeRegistry.cc @@ -17,7 +17,7 @@ void ArrowTypeRegistry::registerType(const std::string& typeName, std::shared_pt } std::shared_ptr ArrowTypeRegistry::getType(const std::string& typeName) const { - ArrowConverterLibraryLoader::instance(); + loadArrowLibraries(); auto it = m_registry.find(typeName); if (it != m_registry.end()) { return it->second; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 759a145af..17dbb250b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,7 @@ SET(core_sources MurmurHash3.cpp SchemaEvolution.cc Glob.cc + LibraryLoader.cc Pythonizations.cc ) diff --git a/src/LibraryLoader.cc b/src/LibraryLoader.cc new file mode 100644 index 000000000..c81127e1e --- /dev/null +++ b/src/LibraryLoader.cc @@ -0,0 +1,91 @@ +#include "podio/utilities/LibraryLoader.h" +#include +#include +#include +#include +#include +#include + +namespace podio { +namespace utilities { + + LibraryLoader::LibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator) : + m_envVarName(std::move(envVarName)), + m_libraryPattern(std::move(libraryPattern)), + m_logDesignator(std::move(logDesignator)) { + + for (const auto& [lib, dir] : getLibNames()) { + const auto status = loadLib(lib, dir); + switch (status) { + case LoadStatus::Success: + std::cerr << "Loaded " << m_logDesignator << " library \'" << lib << "\' (from " << dir << ")" << std::endl; + break; + case LoadStatus::AlreadyLoaded: + std::cerr << m_logDesignator << " library \'" << lib << "\' already loaded. Not loading again from " << dir + << std::endl; + break; + case LoadStatus::Error: { + const char* err = dlerror(); + std::cerr << "ERROR while loading " << m_logDesignator << " library \'" << lib << "\' (from " << dir + << "): " << (err ? err : "Unknown error") << std::endl; + break; + } + } + } + } + + LibraryLoader::LoadStatus LibraryLoader::loadLib(const std::string& libname, const std::string& directory) { + if (m_loadedLibs.find(libname) != m_loadedLibs.end()) { + return LoadStatus::AlreadyLoaded; + } + dlerror(); // Clear any existing error + void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL); + if (libhandle) { + m_loadedLibs.insert({libname, libhandle}); + return LoadStatus::Success; + } + + return LoadStatus::Error; + } + + std::vector> LibraryLoader::getLibNames() const { + namespace fs = std::filesystem; + std::vector> libs; + + const auto ldLibPath = [this]() { + auto pathVar = std::getenv(m_envVarName.c_str()); + if (!pathVar) { + pathVar = std::getenv("LD_LIBRARY_PATH"); + } + return pathVar; + }(); + if (!ldLibPath) { + return libs; + } + + std::string dir; + std::istringstream stream(ldLibPath); + while (std::getline(stream, dir, ':')) { + if (not fs::exists(dir)) { + continue; + } + + for (auto& lib : fs::directory_iterator(dir)) { + const auto filename = lib.path().filename().string(); + if (filename.find(m_libraryPattern) != std::string::npos) { + libs.emplace_back(std::move(filename), dir); + } + } + + if (std::getenv(m_envVarName.c_str()) && libs.empty()) { + throw std::runtime_error("No " + m_logDesignator + " libraries found in " + m_envVarName + ". Please set " + + m_envVarName + " to the directory containing the " + m_logDesignator + + " libraries or unset it to fallback to LD_LIBRARY_PATH."); + } + } + + return libs; + } + +} // namespace utilities +} // namespace podio diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index b8e10cf9c..a89a25535 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -1,9 +1,11 @@ #include "podio/SIOBlock.h" +#include "podio/utilities/LibraryLoader.h" #include #include #include #include +#include #include #include @@ -97,75 +99,8 @@ std::shared_ptr SIOBlockFactory::createBlock(const podio::CollectionBa return nullptr; } } - -SIOBlockLibraryLoader::SIOBlockLibraryLoader() { - for (const auto& [lib, dir] : getLibNames()) { - const auto status = loadLib(lib, dir); - switch (status) { - case LoadStatus::Success: - std::cerr << "Loaded SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl; - break; - case LoadStatus::AlreadyLoaded: - std::cerr << "SIOBlocks library \'" << lib << "\' already loaded. Not loading again from " << dir << std::endl; - break; - case LoadStatus::Error: - std::cerr << "ERROR while loading SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl; - break; - } - } -} - -SIOBlockLibraryLoader::LoadStatus SIOBlockLibraryLoader::loadLib(const std::string& libname, - const std::string& directory) { - if (_loadedLibs.find(libname) != _loadedLibs.end()) { - return LoadStatus::AlreadyLoaded; - } - void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL); - if (libhandle) { - _loadedLibs.insert({libname, libhandle}); - return LoadStatus::Success; - } - - return LoadStatus::Error; -} - -std::vector> SIOBlockLibraryLoader::getLibNames() { - namespace fs = std::filesystem; - std::vector> libs; - - const auto ldLibPath = []() { - // Check PODIO_SIOBLOCK_PATH first and fall back to LD_LIBRARY_PATH - auto pathVar = std::getenv("PODIO_SIOBLOCK_PATH"); - if (!pathVar) { - pathVar = std::getenv("LD_LIBRARY_PATH"); - } - return pathVar; - }(); - if (!ldLibPath) { - return libs; - } - - std::string dir; - std::istringstream stream(ldLibPath); - while (std::getline(stream, dir, ':')) { - if (not fs::exists(dir)) { - continue; - } - - for (auto& lib : fs::directory_iterator(dir)) { - const auto filename = lib.path().filename().string(); - if (filename.find("SioBlocks") != std::string::npos) { - libs.emplace_back(std::move(filename), dir); - } - } - if (std::getenv("PODIO_SIOBLOCK_PATH") && libs.empty()) { - throw std::runtime_error( - "No SIOBlocks libraries found in PODIO_SIOBLOCK_PATH. Please set PODIO_SIOBLOCK_PATH to the directory " - "containing the SIOBlocks libraries or unset it to fallback to LD_LIBRARY_PATH."); - } - } - - return libs; +void loadSIOBlocksLibraries() { + static podio::utilities::LibraryLoader me("PODIO_SIOBLOCK_PATH", "SioBlocks", "SIOBlocks"); } void SIOFileTOCRecord::addRecord(const std::string& name, PositionType startPos) { diff --git a/src/SIOLegacyReader.cc b/src/SIOLegacyReader.cc index 8d857c908..cadba437c 100644 --- a/src/SIOLegacyReader.cc +++ b/src/SIOLegacyReader.cc @@ -9,7 +9,7 @@ namespace podio { SIOLegacyReader::SIOLegacyReader() { - SIOBlockLibraryLoader::instance(); + loadSIOBlocksLibraries(); } void SIOLegacyReader::openFile(const std::string& filename) { diff --git a/src/SIOReader.cc b/src/SIOReader.cc index cac3d3a3c..4b26a116b 100644 --- a/src/SIOReader.cc +++ b/src/SIOReader.cc @@ -11,7 +11,7 @@ namespace podio { SIOReader::SIOReader() { - SIOBlockLibraryLoader::instance(); + loadSIOBlocksLibraries(); } void SIOReader::openFile(const std::string& filename) { diff --git a/src/SIOWriter.cc b/src/SIOWriter.cc index 87e50e611..f8ef16c02 100644 --- a/src/SIOWriter.cc +++ b/src/SIOWriter.cc @@ -16,7 +16,7 @@ SIOWriter::SIOWriter(const std::string& filename) { SIO_THROW(sio::error_code::not_open, "Couldn't open output stream '" + filename + "'"); } - SIOBlockLibraryLoader::instance(); + loadSIOBlocksLibraries(); sio::block_list blocks; blocks.emplace_back(std::make_shared(podio::version::build_version)); From 1b000fc580e58e9362b36259ef4af716a94fd851 Mon Sep 17 00:00:00 2001 From: arnavdham Date: Tue, 11 Aug 2026 17:44:48 +0530 Subject: [PATCH 7/8] resolving comments --- .../{LibraryLoader.h => BackendLibraryLoader.h} | 12 ++++++------ src/ArrowConverterRegistry.cc | 9 ++------- src/{LibraryLoader.cc => BackendLibraryLoader.cc} | 10 ++++++---- src/CMakeLists.txt | 2 +- src/SIOBlock.cc | 10 ++-------- 5 files changed, 17 insertions(+), 26 deletions(-) rename include/podio/utilities/{LibraryLoader.h => BackendLibraryLoader.h} (64%) rename src/{LibraryLoader.cc => BackendLibraryLoader.cc} (84%) diff --git a/include/podio/utilities/LibraryLoader.h b/include/podio/utilities/BackendLibraryLoader.h similarity index 64% rename from include/podio/utilities/LibraryLoader.h rename to include/podio/utilities/BackendLibraryLoader.h index bdfa7a5eb..e7cb1f215 100644 --- a/include/podio/utilities/LibraryLoader.h +++ b/include/podio/utilities/BackendLibraryLoader.h @@ -1,5 +1,5 @@ -#ifndef PODIO_UTILITIES_LIBRARYLOADER_H -#define PODIO_UTILITIES_LIBRARYLOADER_H +#ifndef PODIO_UTILITIES_BACKENDLIBRARYLOADER_H +#define PODIO_UTILITIES_BACKENDLIBRARYLOADER_H #include #include @@ -9,12 +9,12 @@ namespace podio { namespace utilities { - class LibraryLoader { + class BackendLibraryLoader { public: enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 }; - LibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator); - ~LibraryLoader() = default; + BackendLibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator); + ~BackendLibraryLoader() = default; private: LoadStatus loadLib(const std::string& libname, const std::string& directory); @@ -29,4 +29,4 @@ namespace utilities { } // namespace utilities } // namespace podio -#endif // PODIO_UTILITIES_LIBRARYLOADER_H +#endif // PODIO_UTILITIES_BACKENDLIBRARYLOADER_H diff --git a/src/ArrowConverterRegistry.cc b/src/ArrowConverterRegistry.cc index e3dc7b69f..ef0cb858e 100644 --- a/src/ArrowConverterRegistry.cc +++ b/src/ArrowConverterRegistry.cc @@ -1,10 +1,5 @@ #include "podio/utilities/ArrowConverterRegistry.h" -#include "podio/utilities/LibraryLoader.h" -#include -#include -#include -#include -#include +#include "podio/utilities/BackendLibraryLoader.h" namespace podio { @@ -44,7 +39,7 @@ ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const } void loadArrowLibraries() { - static podio::utilities::LibraryLoader me("PODIO_ARROW_PATH", "Arrow", "Arrow"); + static podio::utilities::BackendLibraryLoader me("PODIO_ARROW_PATH", "Arrow", "Arrow"); } } // namespace podio diff --git a/src/LibraryLoader.cc b/src/BackendLibraryLoader.cc similarity index 84% rename from src/LibraryLoader.cc rename to src/BackendLibraryLoader.cc index c81127e1e..2a079eea7 100644 --- a/src/LibraryLoader.cc +++ b/src/BackendLibraryLoader.cc @@ -1,4 +1,4 @@ -#include "podio/utilities/LibraryLoader.h" +#include "podio/utilities/BackendLibraryLoader.h" #include #include #include @@ -9,7 +9,8 @@ namespace podio { namespace utilities { - LibraryLoader::LibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator) : + BackendLibraryLoader::BackendLibraryLoader(std::string envVarName, std::string libraryPattern, + std::string logDesignator) : m_envVarName(std::move(envVarName)), m_libraryPattern(std::move(libraryPattern)), m_logDesignator(std::move(logDesignator)) { @@ -34,7 +35,8 @@ namespace utilities { } } - LibraryLoader::LoadStatus LibraryLoader::loadLib(const std::string& libname, const std::string& directory) { + BackendLibraryLoader::LoadStatus BackendLibraryLoader::loadLib(const std::string& libname, + const std::string& directory) { if (m_loadedLibs.find(libname) != m_loadedLibs.end()) { return LoadStatus::AlreadyLoaded; } @@ -48,7 +50,7 @@ namespace utilities { return LoadStatus::Error; } - std::vector> LibraryLoader::getLibNames() const { + std::vector> BackendLibraryLoader::getLibNames() const { namespace fs = std::filesystem; std::vector> libs; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17dbb250b..254803544 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,7 +58,7 @@ SET(core_sources MurmurHash3.cpp SchemaEvolution.cc Glob.cc - LibraryLoader.cc + BackendLibraryLoader.cc Pythonizations.cc ) diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index a89a25535..7276c9c4e 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -1,13 +1,7 @@ #include "podio/SIOBlock.h" -#include "podio/utilities/LibraryLoader.h" +#include "podio/utilities/BackendLibraryLoader.h" #include -#include -#include -#include -#include -#include -#include namespace podio { @@ -100,7 +94,7 @@ std::shared_ptr SIOBlockFactory::createBlock(const podio::CollectionBa } } void loadSIOBlocksLibraries() { - static podio::utilities::LibraryLoader me("PODIO_SIOBLOCK_PATH", "SioBlocks", "SIOBlocks"); + static podio::utilities::BackendLibraryLoader me("PODIO_SIOBLOCK_PATH", "SioBlocks", "SIOBlocks"); } void SIOFileTOCRecord::addRecord(const std::string& name, PositionType startPos) { From 36c12eb3681eb67db644d0d24a807dfb67ad05c1 Mon Sep 17 00:00:00 2001 From: arnavdham Date: Tue, 11 Aug 2026 23:10:38 +0530 Subject: [PATCH 8/8] Arrow->PodioAroow --- cmake/podioMacros.cmake | 12 +++++------- src/ArrowConverterRegistry.cc | 2 +- tests/unittests/CMakeLists.txt | 6 +++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmake/podioMacros.cmake b/cmake/podioMacros.cmake index c70630e41..7cba9584e 100644 --- a/cmake/podioMacros.cmake +++ b/cmake/podioMacros.cmake @@ -340,7 +340,7 @@ endfunction() # ArrowMapper code has been generated. # # Arguments: -# CORE_LIB The name of the core datamodel library. The name of the Arrow library target will be ${CORE_LIB}Arrow +# CORE_LIB The name of the core datamodel library. The name of the Arrow library target will be ${CORE_LIB}PodioArrow # HEADERS The list of all header files created by PODIO_GENERATE_DATAMODEL # SOURCES The list of all source files created by PODIO_GENERATE_DATAMODEL # @@ -361,12 +361,10 @@ endif() # Only get the ArrowMapper handlers list(FILTER SOURCES INCLUDE REGEX .*ArrowMapper.cc) - add_library(${CORE_LIB}Arrow SHARED ${SOURCES}) - target_link_libraries(${CORE_LIB}Arrow PUBLIC ${CORE_LIB} podio::podio ${PODIO_ARROW_TARGET}) - target_include_directories(${CORE_LIB}Arrow PUBLIC + add_library(${CORE_LIB}PodioArrow SHARED ${SOURCES}) + target_link_libraries(${CORE_LIB}PodioArrow PUBLIC ${CORE_LIB} podio::podio ${PODIO_ARROW_TARGET}) + target_include_directories(${CORE_LIB}PodioArrow PUBLIC $ $) - - # Disable clang-tidy on generated sources - set_target_properties(${CORE_LIB}Arrow PROPERTIES CXX_CLANG_TIDY "") + set_target_properties(${CORE_LIB}PodioArrow PROPERTIES CXX_CLANG_TIDY "") endfunction() diff --git a/src/ArrowConverterRegistry.cc b/src/ArrowConverterRegistry.cc index ef0cb858e..97a208d5e 100644 --- a/src/ArrowConverterRegistry.cc +++ b/src/ArrowConverterRegistry.cc @@ -39,7 +39,7 @@ ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const } void loadArrowLibraries() { - static podio::utilities::BackendLibraryLoader me("PODIO_ARROW_PATH", "Arrow", "Arrow"); + static podio::utilities::BackendLibraryLoader me("PODIO_ARROW_PATH", "PodioArrow", "Arrow"); } } // namespace podio diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 984168a4b..0379c9676 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -54,9 +54,9 @@ if (ENABLE_ARROW) target_sources(unittest_podio PRIVATE test_arrow_converter.cpp) target_link_libraries(unittest_podio PRIVATE podio::podioArrow ${PODIO_ARROW_TARGET}) add_dependencies(unittest_podio - TestDataModelArrow - ExtensionDataModelArrow - InterfaceExtensionDataModelArrow + TestDataModelPodioArrow + ExtensionDataModelPodioArrow + InterfaceExtensionDataModelPodioArrow ) target_compile_definitions(unittest_podio PRIVATE PODIO_ENABLE_ARROW=1) endif()