Issue
We are trying to compile some c++ code into a shared library with pybind11. This shared library does not call python functions, but is instead called from a python script. Then we want to create an entrypoint C++ executable that uses this shared library to test the c++ code. It does not interface with pybind or the python side in any way.
When we do so, our shared library compiles successfully. However, in the executable target, we get a variety of linker errors specifying 'undefined reference to' our constructors in the shared library.
After reading the pybind documentation, we believe this is because the pybind11_add_module function forces cmake to compile the shared library with a 'hidden' visibility flag. We subsequently try to expose the relevant functions in our c++ shared library so that our executable can access it (using __attribute__((visibility("default")))). This resolves the undefined references to our constructors, but results in the errors provided below. Note that these errors occur when compiling our executable. The shared library still compiles succsesfully.
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_SetItem'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_Repr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyInstanceMethod_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_ValueError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyLong_FromSsize_t'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyDict_GetItemString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_TrueStruct'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_IndexError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_NormalizeException'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyInstanceMethod_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyEval_AcquireThread'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_Str'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyThreadState_DeleteCurrent'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyGILState_GetThisThreadState'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_GetAttrString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyModule_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyMem_Free'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_Restore'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyType_IsSubtype'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyModule_AddObject'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_WarnEx'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_SetPointer'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_SetAttr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_IsInstance'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyEval_RestoreThread'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_NoneStruct'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyException_SetTraceback'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_FromFormat'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_Append'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_MemoryError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyType_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyDict_Next'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyTuple_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyBuffer_Release'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyErr_Format'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyObject_CallObject'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyFloat_FromDouble'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_DecodeUTF8'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_Py_Dealloc'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCFunction_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyExc_OverflowError'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCFunction_NewEx'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_New'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyProperty_Type'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `_PyObject_GetDictPtr'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyUnicode_FromString'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `Py_GetVersion'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyCapsule_SetContext'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyFrame_GetLineNumber'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyThread_tss_get'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyBytes_Size'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PySequence_Check'
[build] ../pipeline_manager.cpython-38-x86_64-linux-gnu.so: undefined reference to `PyList_GetItem'
... [additional reference errors removed for brevity]
[build] collect2: error: ld returned 1 exit status
We have tried a variety of commands regarding the visibility of the shared library compilation, but always end up with one or both of the errors described above. Find our CMAKEFILES below:
Main CMakeLists.txt that compiles the shared library (with irrelevant commands for gstreamer, cuda, etc... removed for brevity):
cmake_minimum_required(VERSION 3.5)
project(visual_processing_node)
# Default to C99
if(NOT CMAKE_C_STANDARD)
set(CMAKE_C_STANDARD 99)
endif()
# Default to C++14
if(NOT CMAKE_CXX_STANDARD)
set(CMAKE_CXX_STANDARD 14)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
find_package(Threads REQUIRED)
if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_compile_options(-Wall -Wextra -Wpedantic)
endif()
# Fetch pybind11
include(FetchContent)
FetchContent_Declare(
pybind11
GIT_REPOSITORY https://github.com/pybind/pybind11.git
GIT_TAG v2.6.2
GIT_SHALLOW TRUE
)
FetchContent_MakeAvailable(pybind11)
pybind11_add_module(pipeline_manager SHARED src/deepstream_pipeline.cpp src/deepstream_pipeline_helpers.cpp)
target_include_directories(
pipeline_manager
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include>
${GSTREAMER_INCLUDE_DIRS} ${GSTREAMER_APP_INCLUDE_DIRS}
${GLIB_INCLUDE_DIRS} ${GLOG_INCLUDE_DIRS} "${DeepStream_DIR}/lib")
target_link_libraries(
pipeline_manager
PUBLIC
${GSTREAMER_LIBRARIES}
${GLIB_LIBRARIES}
${GLIB_GIO_LIBRARIES}
${GLIB_GOBJECT_LIBRARIES}
${GLOG_LIBRARIES}
${GSTREAMER_APP_LIBRARIES}
"/opt/nvidia/deepstream/deepstream-6.0/lib/libnvdsgst_meta.so"
"/opt/nvidia/deepstream/deepstream-6.0/lib/libnvds_meta.so"
gstrtspserver-1.0
Threads::Threads)
add_subdirectory (tests)
Second CMakeLists.txt file inside 'tests' directory that compiles the executable.
add_executable (inferenceTest inference_test.cpp)
target_link_libraries (inferenceTest pipeline_manager
pybind11::module)
target_include_directories(inferenceTest PUBLIC
"${PROJECT_BINARY_DIR}")
Solution
I'm not sure what causes the errors you're getting, but I can suggest a workaround, and maybe a better way of doing this.
You could build two shared libraries: one that contains your C++ code you need to test, and a Pybind11 extension that dynamically links to the first one and only provides the binding definitions.
Then you could link with the first one for the tests, without Python getting in the way.
Answered By - unddoch Answer Checked By - Katrina (WPSolving Volunteer)