Refactor backend library loading and rename Arrow suffix to PodioArrow

cmake/podioMacros.cmake

@@ -340,7 +340,7 @@ endfunction()
 # ArrowMapper code has been generated.
 #
 # Arguments:
-#    CORE_LIB             The name of the core datamodel library. The name of the Arrow library target will be ${CORE_LIB}Arrow
+#    CORE_LIB             The name of the core datamodel library. The name of the Arrow library target will be ${CORE_LIB}PodioArrow
 #    HEADERS              The list of all header files created by PODIO_GENERATE_DATAMODEL
 #    SOURCES              The list of all source files created by PODIO_GENERATE_DATAMODEL
 #
@@ -361,12 +361,10 @@ endif()
   # Only get the ArrowMapper handlers
   list(FILTER SOURCES INCLUDE REGEX .*ArrowMapper.cc)
 
-  add_library(${CORE_LIB}Arrow SHARED ${SOURCES})
-  target_link_libraries(${CORE_LIB}Arrow PUBLIC ${CORE_LIB} podio::podio ${PODIO_ARROW_TARGET})
-  target_include_directories(${CORE_LIB}Arrow PUBLIC
+  add_library(${CORE_LIB}PodioArrow SHARED ${SOURCES})
+  target_link_libraries(${CORE_LIB}PodioArrow PUBLIC ${CORE_LIB} podio::podio ${PODIO_ARROW_TARGET})
+  target_include_directories(${CORE_LIB}PodioArrow PUBLIC
     $<BUILD_INTERFACE:${ARG_OUTPUT_FOLDER}>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
-
-  # Disable clang-tidy on generated sources
-  set_target_properties(${CORE_LIB}Arrow PROPERTIES CXX_CLANG_TIDY "")
+  set_target_properties(${CORE_LIB}PodioArrow PROPERTIES CXX_CLANG_TIDY "")
 endfunction()

cmake/podioTest.cmake

@@ -45,6 +45,7 @@ function(PODIO_SET_TEST_ENV test)
     LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/src:$<TARGET_FILE_DIR:ROOT::Tree>:$<$<TARGET_EXISTS:SIO::sio>:$<TARGET_FILE_DIR:SIO::sio>>:$ENV{LD_LIBRARY_PATH}
       PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH}
       PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests
+      PODIO_ARROW_PATH=${PROJECT_BINARY_DIR}/tests
       ROOT_INCLUDE_PATH=${PROJECT_SOURCE_DIR}/tests:${PROJECT_SOURCE_DIR}/include:$ENV{ROOT_INCLUDE_PATH}
       SKIP_SIO_TESTS=$<NOT:$<BOOL:${ENABLE_SIO}>>
       IO_HANDLERS=${IO_HANDLERS}

include/podio/SIOBlock.h

@@ -255,28 +255,7 @@ class SIOBlockFactory {
   }
 };
 
-class SIOBlockLibraryLoader {
-private:
-  SIOBlockLibraryLoader();
-
-  /// Status code for loading shared SIOBlocks libraries
-  enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 };
-
-  /// Load a library with the given name via dlopen
-  LoadStatus loadLib(const std::string& libname, const std::string& directory);
-
-  /// Get all files that are found on LD_LIBRARY_PATH and that have "SioBlocks"
-  /// in their name together with the directory they are in
-  static std::vector<std::tuple<std::string, std::string>> getLibNames();
-
-  std::map<std::string, void*> _loadedLibs{};
-
-public:
-  static SIOBlockLibraryLoader& instance() {
-    static SIOBlockLibraryLoader instance;
-    return instance;
-  }
-};
+void loadSIOBlocksLibraries();
 
 namespace sio_helpers {
   /// marker for showing that a TOC has been stored in the file

include/podio/utilities/ArrowConverterRegistry.h

@@ -20,6 +20,12 @@ class CollectionBase;
 /**
  * @brief Global singleton registry mapping PODIO type name strings to their
  * corresponding Apache Arrow array converter callbacks.
+ *
+ * Registration happens lazily on the first call to getConverter or getReader,
+ * when the necessary datamodel-specific Arrow converter libraries (e.g.,
+ * libedm4hepArrow.so, libTestDataModelArrow.so) are loaded. It is expected that this
+ * happens before worker threads query the registry. Once populated, the registry
+ * is read-only and can be safely accessed from multiple threads concurrently.
  */
 class ArrowConverterRegistry {
 public:
@@ -69,6 +75,8 @@ class ArrowConverterRegistry {
   std::map<std::string, BufferReaderFunc> m_readerRegistry;
 };
 
+void loadArrowLibraries();
+
 } // namespace podio
 
 #endif // PODIO_ARROWCONVERTERREGISTRY_H

include/podio/utilities/ArrowTypeRegistry.h

@@ -15,6 +15,12 @@ namespace podio {
 /**
  * @brief Global singleton registry mapping PODIO type name strings to their
  * corresponding Apache Arrow DataTypes.
+ *
+ * Registration happens lazily on the first call to getType, when the necessary
+ * datamodel-specific Arrow converter libraries (e.g., libedm4hepArrow.so,
+ * libTestDataModelArrow.so) are loaded. It is expected that this happens before
+ * worker threads query the registry. Once populated, the registry is read-only
+ * and can be safely accessed from multiple threads concurrently.
  */
 class ArrowTypeRegistry {
 public:

include/podio/utilities/BackendLibraryLoader.h

@@ -0,0 +1,32 @@
+#ifndef PODIO_UTILITIES_BACKENDLIBRARYLOADER_H
+#define PODIO_UTILITIES_BACKENDLIBRARYLOADER_H
+
+#include <map>
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace podio {
+namespace utilities {
+
+  class BackendLibraryLoader {
+  public:
+    enum class LoadStatus : short { Success = 0, AlreadyLoaded = 1, Error = 2 };
+
+    BackendLibraryLoader(std::string envVarName, std::string libraryPattern, std::string logDesignator);
+    ~BackendLibraryLoader() = default;
+
+  private:
+    LoadStatus loadLib(const std::string& libname, const std::string& directory);
+    std::vector<std::tuple<std::string, std::string>> getLibNames() const;
+
+    std::string m_envVarName;
+    std::string m_libraryPattern;
+    std::string m_logDesignator;
+    std::map<std::string, void*> m_loadedLibs{};
+  };
+
+} // namespace utilities
+} // namespace podio
+
+#endif // PODIO_UTILITIES_BACKENDLIBRARYLOADER_H

src/ArrowConverterRegistry.cc

@@ -1,4 +1,5 @@
 #include "podio/utilities/ArrowConverterRegistry.h"
+#include "podio/utilities/BackendLibraryLoader.h"
 
 namespace podio {
 
@@ -16,6 +17,7 @@ void ArrowConverterRegistry::registerConverter(const std::string& typeName, Crea
 }
 
 ArrowConverterRegistry::CreatorFunc ArrowConverterRegistry::getConverter(const std::string& typeName) const {
+  loadArrowLibraries();
   auto it = m_registry.find(typeName);
   if (it != m_registry.end()) {
     return it->second;
@@ -28,11 +30,16 @@ void ArrowConverterRegistry::registerReader(const std::string& typeName, BufferR
 }
 
 ArrowConverterRegistry::BufferReaderFunc ArrowConverterRegistry::getReader(const std::string& typeName) const {
+  loadArrowLibraries();
   auto it = m_readerRegistry.find(typeName);
   if (it != m_readerRegistry.end()) {
     return it->second;
   }
   return nullptr;
 }
 
+void loadArrowLibraries() {
+  static podio::utilities::BackendLibraryLoader me("PODIO_ARROW_PATH", "PodioArrow", "Arrow");
+}
+
 } // namespace podio

src/ArrowTypeRegistry.cc

@@ -1,4 +1,5 @@
 #include "podio/utilities/ArrowTypeRegistry.h"
+#include "podio/utilities/ArrowConverterRegistry.h"
 
 namespace podio {
 
@@ -16,6 +17,7 @@ void ArrowTypeRegistry::registerType(const std::string& typeName, std::shared_pt
 }
 
 std::shared_ptr<arrow::DataType> ArrowTypeRegistry::getType(const std::string& typeName) const {
+  loadArrowLibraries();
   auto it = m_registry.find(typeName);
   if (it != m_registry.end()) {
     return it->second;

src/BackendLibraryLoader.cc

@@ -0,0 +1,93 @@
+#include "podio/utilities/BackendLibraryLoader.h"
+#include <cstdlib>
+#include <dlfcn.h>
+#include <filesystem>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+
+namespace podio {
+namespace utilities {
+
+  BackendLibraryLoader::BackendLibraryLoader(std::string envVarName, std::string libraryPattern,
+                                             std::string logDesignator) :
+      m_envVarName(std::move(envVarName)),
+      m_libraryPattern(std::move(libraryPattern)),
+      m_logDesignator(std::move(logDesignator)) {
+
+    for (const auto& [lib, dir] : getLibNames()) {
+      const auto status = loadLib(lib, dir);
+      switch (status) {
+      case LoadStatus::Success:
+        std::cerr << "Loaded " << m_logDesignator << " library \'" << lib << "\' (from " << dir << ")" << std::endl;
+        break;
+      case LoadStatus::AlreadyLoaded:
+        std::cerr << m_logDesignator << " library \'" << lib << "\' already loaded. Not loading again from " << dir
+                  << std::endl;
+        break;
+      case LoadStatus::Error: {
+        const char* err = dlerror();
+        std::cerr << "ERROR while loading " << m_logDesignator << " library \'" << lib << "\' (from " << dir
+                  << "): " << (err ? err : "Unknown error") << std::endl;
+        break;
+      }
+      }
+    }
+  }
+
+  BackendLibraryLoader::LoadStatus BackendLibraryLoader::loadLib(const std::string& libname,
+                                                                 const std::string& directory) {
+    if (m_loadedLibs.find(libname) != m_loadedLibs.end()) {
+      return LoadStatus::AlreadyLoaded;
+    }
+    dlerror(); // Clear any existing error
+    void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL);
+    if (libhandle) {
+      m_loadedLibs.insert({libname, libhandle});
+      return LoadStatus::Success;
+    }
+
+    return LoadStatus::Error;
+  }
+
+  std::vector<std::tuple<std::string, std::string>> BackendLibraryLoader::getLibNames() const {
+    namespace fs = std::filesystem;
+    std::vector<std::tuple<std::string, std::string>> libs;
+
+    const auto ldLibPath = [this]() {
+      auto pathVar = std::getenv(m_envVarName.c_str());
+      if (!pathVar) {
+        pathVar = std::getenv("LD_LIBRARY_PATH");
+      }
+      return pathVar;
+    }();
+    if (!ldLibPath) {
+      return libs;
+    }
+
+    std::string dir;
+    std::istringstream stream(ldLibPath);
+    while (std::getline(stream, dir, ':')) {
+      if (not fs::exists(dir)) {
+        continue;
+      }
+
+      for (auto& lib : fs::directory_iterator(dir)) {
+        const auto filename = lib.path().filename().string();
+        if (filename.find(m_libraryPattern) != std::string::npos) {
+          libs.emplace_back(std::move(filename), dir);
+        }
+      }
+
+      if (std::getenv(m_envVarName.c_str()) && libs.empty()) {
+        throw std::runtime_error("No " + m_logDesignator + " libraries found in " + m_envVarName + ". Please set " +
+                                 m_envVarName + " to the directory containing the " + m_logDesignator +
+                                 " libraries or unset it to fallback to LD_LIBRARY_PATH.");
+      }
+    }
+
+    return libs;
+  }
+
+} // namespace utilities
+} // namespace podio

src/CMakeLists.txt

@@ -58,6 +58,7 @@ SET(core_sources
   MurmurHash3.cpp
   SchemaEvolution.cc
   Glob.cc
+  BackendLibraryLoader.cc
   Pythonizations.cc
   )
 

src/SIOBlock.cc

@@ -1,11 +1,7 @@
 #include "podio/SIOBlock.h"
 
+#include "podio/utilities/BackendLibraryLoader.h"
 #include <algorithm>
-#include <cstdlib>
-#include <dlfcn.h>
-#include <filesystem>
-#include <map>
-#include <sstream>
 
 namespace podio {
 
@@ -97,75 +93,8 @@ std::shared_ptr<SIOBlock> SIOBlockFactory::createBlock(const podio::CollectionBa
     return nullptr;
   }
 }
-
-SIOBlockLibraryLoader::SIOBlockLibraryLoader() {
-  for (const auto& [lib, dir] : getLibNames()) {
-    const auto status = loadLib(lib, dir);
-    switch (status) {
-    case LoadStatus::Success:
-      std::cerr << "Loaded SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl;
-      break;
-    case LoadStatus::AlreadyLoaded:
-      std::cerr << "SIOBlocks library \'" << lib << "\' already loaded. Not loading again from " << dir << std::endl;
-      break;
-    case LoadStatus::Error:
-      std::cerr << "ERROR while loading SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl;
-      break;
-    }
-  }
-}
-
-SIOBlockLibraryLoader::LoadStatus SIOBlockLibraryLoader::loadLib(const std::string& libname,
-                                                                 const std::string& directory) {
-  if (_loadedLibs.find(libname) != _loadedLibs.end()) {
-    return LoadStatus::AlreadyLoaded;
-  }
-  void* libhandle = dlopen((directory + "/" + libname).c_str(), RTLD_LAZY | RTLD_GLOBAL);
-  if (libhandle) {
-    _loadedLibs.insert({libname, libhandle});
-    return LoadStatus::Success;
-  }
-
-  return LoadStatus::Error;
-}
-
-std::vector<std::tuple<std::string, std::string>> SIOBlockLibraryLoader::getLibNames() {
-  namespace fs = std::filesystem;
-  std::vector<std::tuple<std::string, std::string>> libs;
-
-  const auto ldLibPath = []() {
-    // Check PODIO_SIOBLOCK_PATH first and fall back to LD_LIBRARY_PATH
-    auto pathVar = std::getenv("PODIO_SIOBLOCK_PATH");
-    if (!pathVar) {
-      pathVar = std::getenv("LD_LIBRARY_PATH");
-    }
-    return pathVar;
-  }();
-  if (!ldLibPath) {
-    return libs;
-  }
-
-  std::string dir;
-  std::istringstream stream(ldLibPath);
-  while (std::getline(stream, dir, ':')) {
-    if (not fs::exists(dir)) {
-      continue;
-    }
-
-    for (auto& lib : fs::directory_iterator(dir)) {
-      const auto filename = lib.path().filename().string();
-      if (filename.find("SioBlocks") != std::string::npos) {
-        libs.emplace_back(std::move(filename), dir);
-      }
-    }
-    if (std::getenv("PODIO_SIOBLOCK_PATH") && libs.empty()) {
-      throw std::runtime_error(
-          "No SIOBlocks libraries found in PODIO_SIOBLOCK_PATH. Please set PODIO_SIOBLOCK_PATH to the directory "
-          "containing the SIOBlocks libraries or unset it to fallback to LD_LIBRARY_PATH.");
-    }
-  }
-
-  return libs;
+void loadSIOBlocksLibraries() {
+  static podio::utilities::BackendLibraryLoader me("PODIO_SIOBLOCK_PATH", "SioBlocks", "SIOBlocks");
 }
 
 void SIOFileTOCRecord::addRecord(const std::string& name, PositionType startPos) {

src/SIOLegacyReader.cc

@@ -9,7 +9,7 @@
 namespace podio {
 
 SIOLegacyReader::SIOLegacyReader() {
-  SIOBlockLibraryLoader::instance();
+  loadSIOBlocksLibraries();
 }
 
 void SIOLegacyReader::openFile(const std::string& filename) {

src/SIOReader.cc

@@ -11,7 +11,7 @@
 namespace podio {
 
 SIOReader::SIOReader() {
-  SIOBlockLibraryLoader::instance();
+  loadSIOBlocksLibraries();
 }
 
 void SIOReader::openFile(const std::string& filename) {

src/SIOWriter.cc

@@ -16,7 +16,7 @@ SIOWriter::SIOWriter(const std::string& filename) {
     SIO_THROW(sio::error_code::not_open, "Couldn't open output stream '" + filename + "'");
   }
 
-  SIOBlockLibraryLoader::instance();
+  loadSIOBlocksLibraries();
 
   sio::block_list blocks;
   blocks.emplace_back(std::make_shared<SIOVersionBlock>(podio::version::build_version));