NVIDIA VisionWorks and stereo_image_proc

stereo_image_proc/CMakeLists.txt

@@ -4,8 +4,65 @@ project(stereo_image_proc)
 find_package(catkin REQUIRED cv_bridge dynamic_reconfigure image_geometry image_proc image_transport message_filters nodelet sensor_msgs stereo_msgs)
 find_package(Boost REQUIRED COMPONENTS thread)
 
+##DEBUGGING BUILD
+#set(CMAKE_VERBOSE_MAKEFILE TRUE)
+
+# CUDA library loading
+#
+# Requires: pkg-config
+#
+# Try to load package-config. If it doesn't exist, then we'll just fail to load CUDA libraries, no big deal.
+set(all_visionworks_INCLUDE_DIRS "")
+set(all_visionworks_LIBRARIES "")
+find_package(PkgConfig QUIET)
+if(PkgConfig_FOUND)
+  #Prefer CUDA 9.0, then 9.1
+  pkg_check_modules(CUDART cudart-9.0)
+  pkg_check_modules(CORE_CUDA cuda-9.0)
+  if(CUDART_FOUND AND CORE_CUDA_FOUND)
+    message(STATUS "Found CUDA 9.0")
+  else()
+    pkg_check_modules(CUDART cudart-9.1)
+    pkg_check_modules(CORE_CUDA cuda-9.1)
+    if(CUDART_FOUND AND CORE_CUDA_FOUND)
+      message(STATUS "Found CUDA 9.1")
+    endif()
+  endif()
+  if(NOT CUDART_FOUND OR NOT CORE_CUDA_FOUND)
+    message(STATUS "No CUDA found.")
+  else()
+    pkg_check_modules(VISIONWORKS visionworks)
+    if(VISIONWORKS_FOUND)
+      # Note about NVXLIB AND OVXLIB
+      # These libraries normally compile statically, without PIC (Position Independent Code). This means that they cannot be
+      # linked into a .so!
+      # Normally, this is fine, because we'd link them directly into the end binary. However, we're using nodelets, which
+      # are loaded at runtime into the "nodelet" binary, which is not linked against NVX or OVX.
+      # Thus, we need them in the .so. So, you MUST compile VisionWorks sources with the CXX flag "-fPIC"!
+      find_library(NVXLIB nvx PATHS ${VISIONWORKS_LIBRARY_DIRS})
+      find_library(OVXLIB ovx PATHS ${VISIONWORKS_LIBRARY_DIRS})
+      if(NOT NVXLIB OR NOT OVXLIB)
+        message(WARNING "Found visionworks, but didn't find libovx or libnvx! You probably need to compile the visionworks sources!")
+      else()
+        message(STATUS "NVX library: ${NVXLIB}")
+        message(STATUS "OVX library: ${OVXLIB}")
+      endif()
+    else()
+      message(WARNING "Found CUDA, but could not find VisionWorks")
+    endif()
+  endif()
+
+  if(CUDART_FOUND AND CORE_CUDA_FOUND AND VISIONWORKS_FOUND AND NVXLIB AND OVXLIB)
+    message(STATUS "Found acceptable CUDA and VisionWorks on this system. Will use GPU acceleration.")
+    set(all_visionworks_FOUND TRUE)
+    add_definitions(-DVISIONWORKS_ACCELERATION)
+  endif()
+else()
+  message(STATUS "Could not find pkg-config, so will not try to use CUDA acceleration.")
+endif()
+
 if(cv_bridge_VERSION VERSION_GREATER "1.12.0")
-  add_compile_options(-std=c++11)
+  add_compile_options(-std=c++14)
 endif()
 
 # Dynamic reconfigure support
@@ -20,16 +77,42 @@ catkin_package(
 include_directories(include)
 
 find_package(OpenCV REQUIRED)
-include_directories(${catkin_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})
+include_directories(${catkin_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS} ${all_visionworks_INCLUDE_DIRS})
+if(all_visionworks_FOUND)
+  include_directories(${CUDART_INCLUDE_DIRS} ${CORE_CUDA_INCLUDE_DIRS} ${VISIONWORKS_INCLUDE_DIRS})
+endif()
 
 # See note in image_proc/CMakeLists.txt
 add_definitions(-DOPENCV_TRAITS_ENABLE_DEPRECATED)
 
 # Nodelet library
-add_library(${PROJECT_NAME} src/libstereo_image_proc/processor.cpp src/nodelets/disparity.cpp src/nodelets/point_cloud2.cpp)
-target_link_libraries(${PROJECT_NAME} ${catkin_LIBRARIES}
-                                      ${OpenCV_LIBRARIES}
-)
+if(all_visionworks_FOUND)
+  add_library(${PROJECT_NAME}_visionworks
+    src/libvisionworks_image_proc/visionworks_interface.cpp
+    src/libvisionworks_image_proc/visionworks_cv_helpers.cpp
+    src/libvisionworks_image_proc/speckle_filter_node.cpp
+    src/libvisionworks_image_proc/masking_node.cpp
+    src/libvisionworks_image_proc/sobel_filter_node.cpp)
+  target_link_libraries(${PROJECT_NAME}_visionworks ${catkin_LIBRARIES}
+                        ${OpenCV_LIBRARIES}
+                        ${CUDART_LDFLAGS}
+                        ${CORE_CUDA_LDFLAGS}
+                        ${VISIONWORKS_LDFLAGS}
+                        ${NVXLIB}
+                        ${OVXLIB}
+                        )
+
+  add_library(${PROJECT_NAME} src/libstereo_image_proc/processor.cpp src/libvisionworks_image_proc/visionworks_matching.cpp src/nodelets/disparity.cpp src/nodelets/point_cloud2.cpp)  
+  target_link_libraries(${PROJECT_NAME} ${catkin_LIBRARIES}
+                        ${OpenCV_LIBRARIES}
+                        ${PROJECT_NAME}_visionworks
+                        )
+else()
+  add_library(${PROJECT_NAME} src/libstereo_image_proc/processor.cpp src/nodelets/disparity.cpp src/nodelets/point_cloud2.cpp)
+  target_link_libraries(${PROJECT_NAME} ${catkin_LIBRARIES}
+                                        ${OpenCV_LIBRARIES}
+                                        )
+endif()
 add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}_gencfg)
 install(TARGETS ${PROJECT_NAME}
         DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
@@ -40,8 +123,8 @@ install(FILES nodelet_plugins.xml
 
 # Standalone node
 add_executable(stereoimageproc_exe src/nodes/stereo_image_proc.cpp)
-target_link_libraries(stereoimageproc_exe stereo_image_proc)
-SET_TARGET_PROPERTIES(stereoimageproc_exe PROPERTIES OUTPUT_NAME stereo_image_proc)
+target_link_libraries(stereoimageproc_exe ${PROJECT_NAME})
+SET_TARGET_PROPERTIES(stereoimageproc_exe PROPERTIES OUTPUT_NAME ${PROJECT_NAME})
 install(TARGETS stereoimageproc_exe
         DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
 )

stereo_image_proc/README.rst

@@ -0,0 +1,125 @@
+=================
+stereo_image_proc
+=================
+
+Original Authors:
+
+* Patrick Mihelich
+* Kurt Konolige
+* Jeremy Leibs
+
+Original Maintainer:
+
+* Vincent Rabaud <vincent.rabaud AT gmail DOT com>
+
+-----------------------
+VisionWorks Integration
+-----------------------
+
+Author:
+
+* Zach LaCelle <[email protected]>
+
+NVIDIA VisionWorks is a set of libraries and APIs enabing
+CUDA-accelerated parallel computing of vision processing
+algorithms. It provides an easy-to-use, optimized method
+of adding parallel compute capabilities to a vision processing
+project.
+
+Documentation of VisionWorks is here (account creation required):
+
+https://developer.nvidia.com/embedded/visionworks
+
+VisionWorks makes use of the OpenVX standard, maintained by
+the Khronos group. Details of the OpenVX specification are
+available here:
+
+https://www.khronos.org/openvx/
+
+VisionWorks is supported on the NVIDIA Drive and NVIDIA TX2
+systems, as well as on traditional graphics cards--making it
+useful for integration with autonomy applications.
+
+Tested Versions
+...............
+
+This software has been tested with VisionWorks 1.6, which includes
+CUDA 9.0.
+
+Installation
+............
+
+To enable VisionWorks support, download VisionWorks via JetPack and
+install locally. This will also install CUDA.
+
+After installation of VisionWorks and CUDA, the VisionWorks interface
+libraries must be custom-compiled.  To link into ROS nodelet libraries,
+these static libs must be compiled with Position-Independent Code (-fPIC).
+
+Manually edit the makefile at /usr/share/visionworks/sources, and add:
+
+``export CXXFLAGS+=-fPIC``
+
+If you don't do this, the nodelet will fail at runtime.
+
+To make Package-Config work with VisionWorks, you must also fix the
+visionworks.pc file located at /usr/lib/pkgconfig/visionworks.pc, replacing
+the corresponding lines with:
+
+::
+
+   prefix=/usr/share/visionworks/sources
+   exec_prefix=${prefix}/bin/<arch>/linux/release
+   libdir=${prefix}/libs/<arch>/linux/release
+   includedir=${prefix}/nvxio/include
+
+Note that you need to replace <arch> with your system's architecture, shown
+with the output of ``uname -m``
+
+At this point, you should be ready to compile stereo_image_proc with
+VisionWorks.
+
+Compiling
+.........
+
+The CMakeLists.txt for stereo_image_proc searches for cudart-X.X and
+cuda-X.X with pkg-config (where X.X is the supported version number)
+and, if found, searches for visionworks with pkg-config. Assuming these
+are found, it then searches for libnvx and libovx, and generates a
+libstereo_image_proc_visionworks.so library linked against these. This
+library is then linked into libstereo_image_proc.so. It also passes
+-DVISIONWORKS_ACCELERATION to the compiler, which is used in
+precompiler directives to enable VisionWorks blocks of code.
+
+If these packages and libraries are not found, it continues compiling
+without support for VisionWorks.
+
+Generating Documentation
+------------------------
+
+VisionWorks integration is documented through Doxygen comments.
+
+To generate Doxygen documentation, run the following command from the
+stereo_image_proc directory:
+
+::
+
+   rosdoc_lite .
+
+The output will be in doc/html/c++. To open it, run:
+
+::
+
+   firefox doc/html/c++/index.html
+
+For more information, see the rosdoc_lite documentation here:
+
+http://wiki.ros.org/rosdoc_lite      
+
+Enabling VisionWorks
+--------------------
+
+VisionWorks is enable-able at runtime using ROS dynamic-reconfigure. The
+corresponding flag is "use_visionworks." Set the flag to True or check
+the box to switch to running VisionWorks-enabled code. Note that you
+can switch at runtime without restarting.

stereo_image_proc/cfg/Disparity.cfg

@@ -36,4 +36,9 @@ gen.add("disp12MaxDiff", int_t, 0, "Maximum allowed difference (in integer pixel
 # First string value is node name, used only for generating documentation
 # Second string value ("Disparity") is name of class and generated
 #    .h file, with "Config" added, so class DisparityConfig
+
+# CUDA acceleration
+gen.add("use_visionworks",           bool_t,  0, "If compiled in, will use NVIDIA VisionWorks API for CUDA acceleration", False)
+gen.add("print_stereo_perf",    bool_t,  0, "Print disparity pipeline performance information", False)
+
 exit(gen.generate(PACKAGE, "stereo_image_proc", "Disparity"))

stereo_image_proc/doc/.gitignore

@@ -0,0 +1 @@
+html