From 4c45b4b4a290ba961b8fdb5cb83d97f22d9adb5b Mon Sep 17 00:00:00 2001 From: Michael Lingelbach Date: Thu, 9 Sep 2021 16:44:43 -0700 Subject: [PATCH] feat: add linux vr support (#481) --- docs/virtual_reality.md | 7 +- igibson/render/CMakeLists.txt | 93 ++++++++++++------- igibson/render/cpp/glfw_mesh_renderer.h | 2 - igibson/render/cpp/mesh_renderer.h | 4 +- igibson/render/cpp/vr_mesh_renderer.cpp | 54 ++++++++--- igibson/render/cpp/vr_mesh_renderer.h | 10 +- .../render/mesh_renderer/mesh_renderer_cpu.py | 15 ++- .../render/mesh_renderer/mesh_renderer_vr.py | 5 +- igibson/utils/ig_logging.py | 1 + setup.py | 4 + 10 files changed, 135 insertions(+), 60 deletions(-) diff --git a/docs/virtual_reality.md b/docs/virtual_reality.md index fb4e9a4a8..f55a89d4e 100644 --- a/docs/virtual_reality.md +++ b/docs/virtual_reality.md @@ -1,15 +1,16 @@ ### Virtual Reality Overview -Virtual reality is currently only supported on Windows 10/11 on the HTC Vive and Oculus Rift/Quest (with link). +Virtual reality is currently supported on Windows 10/11 on the HTC Vive and Oculus Rift/Quest (with link), and on Linux with the HTC Vive (there is no linux driver for Oculus). -Linux support is planned, but the HTC Vive Pro Eye tracking driver is not available for Linux, and currently aysnchronous projection/motion smoothing is not supported on the Nvidia proprietary drivers on linux. +The HTC Vive Pro Eye tracking driver is not available for Linux. You must have the latest Nvidia driver 470.XX and SteamVR 1.19.7 as asynchronous reprojection (a form of motion smoothing necessary for a usable VR experience) was only added in mid 2021. ### Setup 1. Set up the HTC Vive VR hardware according to the [setup guide](https://support.steampowered.com/steamvr/HTC_Vive/) 2. (optional) if you plan to use eye tracking on Windows, create a [vive developer account](https://hub.vive.com/sso/login) then download and install the [SRAnipal runtime](https://developer.vive.com/resources/vive-sense/sdk/vive-eye-and-facial-tracking-sdk/). Note you should [calibrate](https://developer.vive.com/us/support/sdk/category_howto/how-to-calibrate-eye-tracking.html) the Vive eye tracker before each recording session. -3. Ensure you have installed iGibson according to the installation [instructions](http://svl.stanford.edu/igibson/docs/installation.html#installation-method) +3. Ensure you have installed iGibson according to the installation [instructions](http://svl.stanford.edu/igibson/docs/installation.html#installation-method). + * Note: On Windows VR support is eanbled by default. On Linux, you must install with an additional environmental variable `USE_VR=True pip install -e .`. You must also have addition development headers installed: on Ubuntu `sudo apt install xorg-dev` and on Centos/Fedora: `sudo dnf install libXinerama-devel libXi-devel libXrandr-devel libXcursor-devel`. ### VR examples diff --git a/igibson/render/CMakeLists.txt b/igibson/render/CMakeLists.txt index 65f5ba714..738538113 100644 --- a/igibson/render/CMakeLists.txt +++ b/igibson/render/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 2.8.12) project(CppMeshRenderer) set(USE_GLAD TRUE) +set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +set(OpenGL_GL_PREFERENCE GLVND) if (MAC_PLATFORM) set(USE_CUDA FALSE) @@ -12,6 +14,10 @@ elseif (WIN32) set(USE_GLFW TRUE) # iGibson on Windows is always in VR mode set(USE_VR TRUE) +elseif (UNIX AND USE_VR) + set(USE_CUDA TRUE) + set(USE_GLFW TRUE) + set(USE_VR TRUE) else () set(USE_CUDA TRUE) set(USE_GLFW FALSE) @@ -57,53 +63,68 @@ endif () if (USE_VR) # Find OpenVR - set(WINDOWS_PATH_SUFFIXES win64 Win64 x64) set(OPENVR_DIR openvr) - find_library(OPENVR_LIBRARIES - NAMES - openvr_api - PATHS - "${OPENVR_DIR}/bin" - "${OPENVR_DIR}/lib" - PATH_SUFFIXES - ${WINDOWS_PATH_SUFFIXES} - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH - ) set(OPENVR_INCLUDE_DIR "${OPENVR_DIR}/headers") include_directories("${OPENVR_INCLUDE_DIR}") - # Find SRAnipal - set(SRANI_DIR sranipal) - find_library(SRANI_LIBRARIES - NAMES - SRAnipal - PATHS - "${SRANI_DIR}/lib" - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH - ) - include_directories("${SRANI_DIR}/include") + if (WIN32) + set(PATH_SUFFIXES win64 Win64 x64) + # Find SRAnipal + set(SRANI_DIR sranipal) + find_library(SRANI_LIBRARIES + NAMES + SRAnipal + PATHS + "${SRANI_DIR}/lib" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + include_directories("${SRANI_DIR}/include") + else () + set(PATH_SUFFIXES linux64 Linux64 x64) + endif() + + find_library(OPENVR_LIBRARIES + NAMES + openvr_api + PATHS + "${OPENVR_DIR}/bin" + "${OPENVR_DIR}/lib" + PATH_SUFFIXES + ${PATH_SUFFIXES} + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + endif() add_library(tinyobjloader MODULE cpp/tinyobjloader/tiny_obj_loader.cc cpp/tinyobjloader/tiny_obj_loader_decrypt.cc cpp/tinyobjloader/bindings.cc) if (USE_VR) add_library(VRRendererContext MODULE glad/gl.cpp cpp/vr_mesh_renderer.cpp cpp/glfw_mesh_renderer.cpp cpp/mesh_renderer.cpp) -else() - if (USE_CUDA) - find_package(CUDA REQUIRED) - set(CUDA_LIBRARIES PUBLIC ${CUDA_LIBRARIES}) - cuda_add_library(EGLRendererContext MODULE glad/egl.cpp glad/gl.cpp cpp/mesh_renderer.cpp cpp/egl_mesh_renderer.cpp) - else () - add_library(EGLRendererContext MODULE glad/egl.cpp glad/gl.cpp cpp/mesh_renderer.cpp cpp/egl_mesh_renderer.cpp) - endif () - if (USE_GLFW) - add_library(GLFWRendererContext MODULE glad/gl.cpp cpp/glfw_mesh_renderer.cpp cpp/mesh_renderer.cpp) - endif () endif() +if (USE_CUDA) + find_package(CUDA REQUIRED) + set(CUDA_LIBRARIES PUBLIC ${CUDA_LIBRARIES}) + cuda_add_library(EGLRendererContext MODULE glad/egl.cpp glad/gl.cpp cpp/mesh_renderer.cpp cpp/egl_mesh_renderer.cpp) +else () + add_library(EGLRendererContext MODULE glad/egl.cpp glad/gl.cpp cpp/mesh_renderer.cpp cpp/egl_mesh_renderer.cpp) +endif () + +if (USE_GLFW) + add_library(GLFWRendererContext MODULE glad/gl.cpp cpp/glfw_mesh_renderer.cpp cpp/mesh_renderer.cpp) +endif () + if (USE_VR) - target_link_libraries(VRRendererContext PRIVATE pybind11::module ${CMAKE_DL_LIBS} glfw ${GLFW_LIBRARIES} ${OPENGL_LIBRARIES} ${OPENVR_LIBRARIES} ${SRANI_LIBRARIES} cryptopp-static) + if (WIN32) + target_link_libraries(VRRendererContext PRIVATE pybind11::module ${CMAKE_DL_LIBS} glfw ${GLFW_LIBRARIES} ${OPENGL_LIBRARIES} ${OPENVR_LIBRARIES} ${SRANI_LIBRARIES} cryptopp-static) + else () + target_link_libraries(VRRendererContext PRIVATE pybind11::module ${CMAKE_DL_LIBS} glfw ${GLFW_LIBRARIES} ${OPENGL_LIBRARIES} ${OPENVR_LIBRARIES} ${CUDA_LIBRARIES} cryptopp-static) + target_link_libraries(EGLRendererContext PRIVATE pybind11::module dl pthread) + # if (USE_GLFW) + target_link_libraries(GLFWRendererContext PRIVATE pybind11::module dl glfw ${GLFW_LIBRARIES} ${OPENGL_LIBRARIES}) + # endif () + endif () else () if (USE_GLAD) target_link_libraries(EGLRendererContext PRIVATE pybind11::module dl pthread cryptopp-static) @@ -135,7 +156,7 @@ endif() set_target_properties(tinyobjloader PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}") -if (NOT USE_VR) +if (UNIX) add_executable(query_devices glad/egl.cpp glad/gl.cpp cpp/query_devices.cpp) add_executable(test_device glad/egl.cpp glad/gl.cpp cpp/test_device.cpp) if (USE_GLAD) diff --git a/igibson/render/cpp/glfw_mesh_renderer.h b/igibson/render/cpp/glfw_mesh_renderer.h index 3fbf4b0dd..cff58aabe 100644 --- a/igibson/render/cpp/glfw_mesh_renderer.h +++ b/igibson/render/cpp/glfw_mesh_renderer.h @@ -2,9 +2,7 @@ #define GLFW_MESH_RENDERER_HEADER #include "mesh_renderer.h" -#ifdef WIN32 #include -#endif class GLFWRendererContext: public MeshRendererContext { diff --git a/igibson/render/cpp/mesh_renderer.h b/igibson/render/cpp/mesh_renderer.h index 52bfe4471..f74b2a1c9 100644 --- a/igibson/render/cpp/mesh_renderer.h +++ b/igibson/render/cpp/mesh_renderer.h @@ -89,7 +89,7 @@ public: cudaGraphicsResource* cuda_res[MAX_NUM_RESOURCES]; #endif - int init() {}; + void init() {}; void release() {}; @@ -254,4 +254,4 @@ public: }; -#endif \ No newline at end of file +#endif diff --git a/igibson/render/cpp/vr_mesh_renderer.cpp b/igibson/render/cpp/vr_mesh_renderer.cpp index 88bfd4094..214a9830e 100644 --- a/igibson/render/cpp/vr_mesh_renderer.cpp +++ b/igibson/render/cpp/vr_mesh_renderer.cpp @@ -1,7 +1,10 @@ #include "vr_mesh_renderer.h" + +#ifdef WIN32 #include "SRanipal.h" #include "SRanipal_Eye.h" #include "SRanipal_Enums.h" +#endif #include #include @@ -159,7 +162,7 @@ py::list VRRendererContext::getDeviceCoordinateSystem(char* device) { // TIMELINE: Call after getDataForVRDevice, since this relies on knowing latest HMD transform py::list VRRendererContext::getEyeTrackingData() { py::list eyeData; - + #ifdef WIN32 // Transform data into Gibson coordinate system before returning to user glm::vec3 gibOrigin(vrToGib * glm::vec4(eyeTrackingData.origin, 1.0)); glm::vec3 gibDir(vrToGib * glm::vec4(eyeTrackingData.dir, 1.0)); @@ -185,10 +188,29 @@ py::list VRRendererContext::getEyeTrackingData() { eyeData.append(dir); eyeData.append(eyeTrackingData.leftPupilDiameter); eyeData.append(eyeTrackingData.rightPupilDiameter); - + // Return dummy data with false validity if eye tracking is not enabled (on non-Windows system) + #else + py::list dummy_origin, dummy_dir; + float dummy_diameter_l, dummy_diameter_r; + eyeData.append(false); + eyeData.append(dummy_origin); + eyeData.append(dummy_dir); + eyeData.append(dummy_diameter_l); + eyeData.append(dummy_diameter_r); + #endif return eyeData; } +// Returns whether the current VR system supports eye tracking +bool VRRendererContext::hasEyeTrackingSupport() { + #ifdef WIN32 + return ViveSR::anipal::Eye::IsViveProEye(); + // Non-windows OS always have eye tracking disabled + #else + return false; + #endif +} + // Gets the VR offset vector in form x, y, z // TIMELINE: Can call any time py::list VRRendererContext::getVROffset() { @@ -202,11 +224,6 @@ py::list VRRendererContext::getVROffset() { return offset; } -// Returns whether the current VR system supports eye tracking -bool VRRendererContext::hasEyeTrackingSupport() { - return ViveSR::anipal::Eye::IsViveProEye(); -} - // Initialize the VR system and compositor // TIMELINE: Call during init of renderer, before height/width are set void VRRendererContext::initVR(bool useEyeTracking) { @@ -238,12 +255,15 @@ void VRRendererContext::initVR(bool useEyeTracking) { // No VR system offset by default vrOffsetVec = glm::vec3(0, 0, 0); + // Only activate eye tracking on Windows + #ifdef WIN32 // Set eye tracking boolean this->useEyeTracking = useEyeTracking; if (useEyeTracking) { shouldShutDownEyeTracking = false; initAnipal(); } + #endif } // Polls for VR events, such as button presses @@ -353,10 +373,12 @@ void VRRendererContext::releaseVR() { vr::VR_Shutdown(); m_pHMD = NULL; + #ifdef WIN32 if (this->useEyeTracking) { this->shouldShutDownEyeTracking = true; eyeTrackingThread->join(); } + #endif } // Sets the offset of the VR headset @@ -410,8 +432,8 @@ void VRRendererContext::updateVRData() { hmdData.index = idx; hmdData.isValidData = true; hmdActualPos = getPositionFromSteamVRMatrix(transformMat); - - setSteamVRMatrixPos(hmdActualPos + vrOffsetVec, transformMat); + glm::vec3 steamVrMatrixPos = hmdActualPos + vrOffsetVec; + setSteamVRMatrixPos(steamVrMatrixPos, transformMat); hmdData.deviceTransform = convertSteamVRMatrixToGlmMat4(transformMat); hmdData.devicePos = getPositionFromSteamVRMatrix(transformMat); @@ -447,7 +469,8 @@ void VRRendererContext::updateVRData() { leftControllerData.isValidData = getControllerDataResult; glm::vec3 leftControllerPos = getPositionFromSteamVRMatrix(transformMat); - setSteamVRMatrixPos(leftControllerPos + vrOffsetVec, transformMat); + glm::vec3 steamVrMatrixPos = leftControllerPos + vrOffsetVec; + setSteamVRMatrixPos(steamVrMatrixPos, transformMat); leftControllerData.deviceTransform = convertSteamVRMatrixToGlmMat4(transformMat); leftControllerData.devicePos = getPositionFromSteamVRMatrix(transformMat); @@ -463,7 +486,9 @@ void VRRendererContext::updateVRData() { rightControllerData.isValidData = getControllerDataResult; glm::vec3 rightControllerPos = getPositionFromSteamVRMatrix(transformMat); - setSteamVRMatrixPos(rightControllerPos + vrOffsetVec, transformMat); + glm::vec3 steamVrMatrixPos = rightControllerPos + vrOffsetVec; + + setSteamVRMatrixPos(steamVrMatrixPos, transformMat); rightControllerData.deviceTransform = convertSteamVRMatrixToGlmMat4(transformMat); rightControllerData.devicePos = getPositionFromSteamVRMatrix(transformMat); @@ -481,7 +506,8 @@ void VRRendererContext::updateVRData() { // Apply VR offset to tracker position glm::vec3 trackerPos = getPositionFromSteamVRMatrix(transformMat); - setSteamVRMatrixPos(trackerPos + vrOffsetVec, transformMat); + glm::vec3 steamVrMatrixPos = trackerPos + vrOffsetVec; + setSteamVRMatrixPos(steamVrMatrixPos, transformMat); if (this->trackerNamesToData.find(serial) != this->trackerNamesToData.end()) { this->trackerNamesToData[serial].index = idx; @@ -641,6 +667,7 @@ glm::vec3 VRRendererContext::getVec3ColFromMat4(int col_index, glm::mat4& mat) { return v; } +#ifdef WIN32 // Initializes the SRAnipal runtime, if the user selects this option void VRRendererContext::initAnipal() { if (!ViveSR::anipal::Eye::IsViveProEye()) { @@ -720,6 +747,7 @@ void VRRendererContext::pollAnipal() { } } } +#endif // Print string version of mat4 for debugging purposes void VRRendererContext::printMat4(glm::mat4& m) { @@ -873,8 +901,8 @@ PYBIND11_MODULE(VRRendererContext, m) { pymodule.def("getDataForVRTracker", &VRRendererContext::getDataForVRTracker); pymodule.def("getDeviceCoordinateSystem", &VRRendererContext::getDeviceCoordinateSystem); pymodule.def("getEyeTrackingData", &VRRendererContext::getEyeTrackingData); - pymodule.def("getVROffset", &VRRendererContext::getVROffset); pymodule.def("hasEyeTrackingSupport", &VRRendererContext::hasEyeTrackingSupport); + pymodule.def("getVROffset", &VRRendererContext::getVROffset); pymodule.def("initVR", &VRRendererContext::initVR); pymodule.def("pollVREvents", &VRRendererContext::pollVREvents); pymodule.def("postRenderVRForEye", &VRRendererContext::postRenderVRForEye); diff --git a/igibson/render/cpp/vr_mesh_renderer.h b/igibson/render/cpp/vr_mesh_renderer.h index f0ea2761b..2e1f7035e 100644 --- a/igibson/render/cpp/vr_mesh_renderer.h +++ b/igibson/render/cpp/vr_mesh_renderer.h @@ -2,9 +2,11 @@ #define VR_MESH_RENDERER_HEADER #include "glfw_mesh_renderer.h" +#ifdef WIN32 #include "SRanipal.h" #include "SRanipal_Eye.h" #include "SRanipal_Enums.h" +#endif #include @@ -75,12 +77,16 @@ public: glm::mat4 gibToVr; glm::mat4 vrToGib; - // SRAnipal variables bool useEyeTracking; + bool shouldShutDownEyeTracking; + + #ifdef WIN32 + // SRAnipal variables std::thread* eyeTrackingThread; ViveSR::anipal::Eye::EyeData eyeData; + #endif + int result; - bool shouldShutDownEyeTracking; // Struct storing eye data for SR anipal - we only return origin and direction in world space // As most users will want to use this ray to query intersection or something similar diff --git a/igibson/render/mesh_renderer/mesh_renderer_cpu.py b/igibson/render/mesh_renderer/mesh_renderer_cpu.py index b70b9bcfa..6e1528189 100644 --- a/igibson/render/mesh_renderer/mesh_renderer_cpu.py +++ b/igibson/render/mesh_renderer/mesh_renderer_cpu.py @@ -143,10 +143,23 @@ class MeshRenderer(object): self.rendering_settings.show_glfw_window, rendering_settings.fullscreen, ) - else: + elif self.platform == "Linux" and self.__class__.__name__ == "MeshRendererVR": + from igibson.render.mesh_renderer import VRRendererContext + + self.r = VRRendererContext.VRRendererContext( + width, + height, + int(self.rendering_settings.glfw_gl_version[0]), + int(self.rendering_settings.glfw_gl_version[1]), + self.rendering_settings.show_glfw_window, + rendering_settings.fullscreen, + ) + elif self.platform == "Linux": from igibson.render.mesh_renderer import EGLRendererContext self.r = EGLRendererContext.EGLRendererContext(width, height, device) + else: + Exception("Unsupported platform and renderer combination") self.r.init() diff --git a/igibson/render/mesh_renderer/mesh_renderer_vr.py b/igibson/render/mesh_renderer/mesh_renderer_vr.py index ee7e9086e..d1008cfac 100644 --- a/igibson/render/mesh_renderer/mesh_renderer_vr.py +++ b/igibson/render/mesh_renderer/mesh_renderer_vr.py @@ -261,7 +261,10 @@ class MeshRendererVR(MeshRenderer): self.width = 1296 self.height = 1440 super().__init__( - width=self.width, height=self.height, rendering_settings=self.vr_rendering_settings, simulator=simulator + width=self.width, + height=self.height, + rendering_settings=self.vr_rendering_settings, + simulator=simulator, ) # Rename self.r to self.vrsys diff --git a/igibson/utils/ig_logging.py b/igibson/utils/ig_logging.py index 4bd297b61..280a928bc 100644 --- a/igibson/utils/ig_logging.py +++ b/igibson/utils/ig_logging.py @@ -398,6 +398,7 @@ class IGLogWriter(object): vr_pos_data.extend(p.getConstraintState(self.vr_robot.parts["body"].movement_cid)) self.data_map["vr"]["vr_device_data"]["vr_position_data"][self.frame_counter, ...] = np.array(vr_pos_data) + # On systems where eye tracking is not supported, we get dummy data and a guaranteed False validity reading is_valid, origin, dir, left_pupil_diameter, right_pupil_diameter = self.sim.get_eye_tracking_data() if is_valid: eye_data_list = [is_valid] diff --git a/setup.py b/setup.py index 8f1001665..f8b47565a 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import codecs +import os import os.path import platform import re @@ -87,6 +88,9 @@ class CMakeBuild(build_ext): else: cmake_args += ["-DMAC_PLATFORM=FALSE"] + if os.getenv("USE_VR"): + cmake_args += ["-DUSE_VR=TRUE"] + cfg = "Debug" if self.debug else "Release" build_args = ["--config", cfg]