From af9749e437bf1616ccd0e73f772cf49abbca1355 Mon Sep 17 00:00:00 2001 From: hzyjerry Date: Thu, 19 Oct 2017 02:43:38 -0700 Subject: [PATCH] OpenGL rendering can now be caught and killed (not orphaned) --- data/decode/package.json | 40 ----- depth/.gitignore | 21 --- depth/.hgignore | 15 -- realenv/core/build.sh | 4 - realenv/core/channels/depth_render.py | 9 -- realenv/core/channels/depth_render/render.cpp | 41 ++--- realenv/core/engine.py | 131 ++++++++++++++++ realenv/core/render/show_3d2.py | 19 +-- realenv/core/test_engine.py | 0 realenv/core/tests/test_engine.py | 8 + realenv/envs/simple_env.py | 78 +--------- realenv/error.py | 140 ++++++++++++++++++ 12 files changed, 304 insertions(+), 202 deletions(-) delete mode 100644 data/decode/package.json delete mode 100644 depth/.gitignore delete mode 100644 depth/.hgignore delete mode 100755 realenv/core/build.sh delete mode 100644 realenv/core/channels/depth_render.py create mode 100644 realenv/core/engine.py create mode 100644 realenv/core/test_engine.py create mode 100644 realenv/core/tests/test_engine.py create mode 100644 realenv/error.py diff --git a/data/decode/package.json b/data/decode/package.json deleted file mode 100644 index b4f2e991a..000000000 --- a/data/decode/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "decodejs", - "version": "0.1.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "bytebuffer": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/bytebuffer/-/bytebuffer-5.0.1.tgz", - "integrity": "sha1-WC7qSxqHO20CCkjVjfhfC7ps/d0=", - "requires": { - "long": "3.2.0" - } - }, - "long": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", - "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s=" - }, - "minimist": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", - "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" - }, - "optimist": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "requires": { - "minimist": "0.0.10", - "wordwrap": "0.0.3" - } - }, - "wordwrap": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" - } - } -} diff --git a/depth/.gitignore b/depth/.gitignore deleted file mode 100644 index 5e2e87f24..000000000 --- a/depth/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -**/*.dll -**/*.exe -build/* -bin-*/* -distrib/build_*/* -**.bak -**.orig -screenshot.bmp -*.pyc -Thumbs.db -OpenGL-tutorial_v* -**.mtl -.DS_Store -build/ -distrib/ -*.obj -depth_render -glew-1.13.0/ -glfw-3.1.2/ -/11HB6XZSh1Q -/19Em1uen8Ae \ No newline at end of file diff --git a/depth/.hgignore b/depth/.hgignore deleted file mode 100644 index 3ad22a081..000000000 --- a/depth/.hgignore +++ /dev/null @@ -1,15 +0,0 @@ -syntax: glob - -*.dll -*.exe -build/* -bin-*/* -distrib/build_*/* -*.bak -*.orig -screenshot.bmp -*.pyc -Thumbs.db -glob:OpenGL-tutorial_v* -relre:.*\.blend.+ -glob:*.mtl \ No newline at end of file diff --git a/realenv/core/build.sh b/realenv/core/build.sh deleted file mode 100755 index b2165bc45..000000000 --- a/realenv/core/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -g++ -std=c++11 ./realenv/envs/render.cpp -o ./realenv/envs/render.so -shared -fPIC -O2 -D_GLIBCXX_USE_CXX11_ABI=0 - - diff --git a/realenv/core/channels/depth_render.py b/realenv/core/channels/depth_render.py deleted file mode 100644 index 5eb34178d..000000000 --- a/realenv/core/channels/depth_render.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -from realenv.data.datasets import get_model_path - -def run_depth_render(): - model_path = get_model_path() - dr_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'depth_render') - os.chdir(dr_path) - print("./depth_render --modelpath {}".format(model_path)) - os.system("./depth_render --modelpath {}".format(model_path)) \ No newline at end of file diff --git a/realenv/core/channels/depth_render/render.cpp b/realenv/core/channels/depth_render/render.cpp index 96710bb4a..fe1162a44 100755 --- a/realenv/core/channels/depth_render/render.cpp +++ b/realenv/core/channels/depth_render/render.cpp @@ -187,16 +187,12 @@ glm::mat4 str_to_mat(std::string str) { glm::mat4 mat = glm::mat4(); std::string delimiter = " "; - //std::cout << "Inside str_to_mat" << str << std::endl; - size_t pos = 0; size_t idx = 0; std::string token; while ((pos = str.find(delimiter)) != std::string::npos) { token = str.substr(0, pos); - //std::cout << token << std::endl; mat[idx % 4][idx / 4] = std::stof(token); - //std::cout << "after " << std::stof(token) << " " << idx % 4 << " " << idx / 4 << std::endl; str.erase(0, pos + delimiter.length()); idx += 1; } @@ -215,7 +211,6 @@ std::vector str_to_vec(std::string str) { while ((pos = str.find(delimiter)) != std::string::npos) { token = str.substr(0, pos); longs.push_back(std::stoul(token)); - //std::cout << "after " << std::stof(token) << " " << idx % 4 << " " << idx / 4 << std::endl; str.erase(0, pos + delimiter.length()); idx += 1; } @@ -578,7 +573,6 @@ int main( int argc, char * argv[] ) std::vector cubeMapCoordToPanoCoord; for(size_t ycoord = 0; ycoord < panoHeight; ycoord++){ - // std::cout << ycoord << std::endl; for(size_t xcoord = 0; xcoord < panoWidth; xcoord++){ size_t ind = reordering[ycoord][xcoord][0]; size_t corrx = reordering[ycoord][xcoord][1]; @@ -601,39 +595,27 @@ int main( int argc, char * argv[] ) do{ - std::cout << "Waiting for incoming task" << std::endl; + std::cout << "Realenv Channel Renderer: waiting for pose" << std::endl; // Wait for next request from client socket.recv (&request); - - std::cout << "Received Hello " << request.data() << std::endl; boost::timer t; - //printf("%s\n", request.data()); - + std::string request_str = std::string(static_cast(request.data()), request.size()); - std::cout << "\nFinished cast" << std::endl; - //std::cout << request_str << std::endl; - glm::mat4 viewMat = str_to_mat(request_str); // Measure speed - //double currentTime = glfwGetTime(); - + // double currentTime = glfwGetTime(); double currentTime = 0; nbFrames++; - if ( currentTime - lastTime >= 1.0 ){ // If last prinf() was more than 1sec ago - // printf and reset + if ( currentTime - lastTime >= 1.0 ){ printf("%f ms/frame %d fps\n", 1000.0/double(nbFrames), nbFrames); nbFrames = 0; lastTime += 1.0; } - //zmq::message_t reply (windowWidth*windowHeight*sizeof(unsigned short) * 6); - - //std::cout << "message reply size " << windowWidth*windowHeight*sizeof(float) * 6 << std::endl; - glBindFramebuffer(GL_FRAMEBUFFER, FramebufferName); glViewport(0,0,windowWidth,windowHeight); // Render on the whole framebuffer, complete from the lower left corner to the upper right @@ -641,7 +623,7 @@ int main( int argc, char * argv[] ) //int nByte = nSize*sizeof(unsigned short); int nByte = nSize*sizeof(float); - // First let's create our buffer, 3 channels per Pixel + // create buffer, 3 channels per Pixel float* dataBuffer = (float*)malloc(nByte); //char* dataBuffer = (char*)malloc(nSize*sizeof(char)); @@ -661,18 +643,15 @@ int main( int argc, char * argv[] ) glUseProgram(programID); // Compute the MVP matrix from keyboard and mouse input - //computeMatricesFromInputs(); - //computeMatricesFromFile(name_loc); + // computeMatricesFromInputs(); + // computeMatricesFromFile(name_loc); float fov = glm::radians(90.0f); - glm::mat4 ProjectionMatrix = glm::perspective(fov, 1.0f, 0.1f, 5000.0f); // near & far are not verified, but accuracy seems to work well + glm::mat4 ProjectionMatrix = glm::perspective(fov, 1.0f, 0.1f, 5000.0f); + // TODO: (hzyjerry) near & far are not verified, but accuracy seems to work well glm::mat4 ViewMatrix = getView(viewMat, k); //glm::mat4 ViewMatrix = getViewMatrix(); glm::mat4 viewMatPose = glm::inverse(ViewMatrix); - // printf("View (pose) matrix for skybox %d\n", k); - // for (int i = 0; i < 4; ++i) { - // printf("\t %f %f %f %f\n", viewMatPose[0][i], viewMatPose[1][i], viewMatPose[2][i], viewMatPose[3][i]); - // //printf("\t %f %f %f %f\n", ViewMatrix[0][i], ViewMatrix[1][i], ViewMatrix[2][i], ViewMatrix[3][i]); - // } + glm::mat4 ModelMatrix = glm::mat4(1.0); diff --git a/realenv/core/engine.py b/realenv/core/engine.py new file mode 100644 index 000000000..171052a55 --- /dev/null +++ b/realenv/core/engine.py @@ -0,0 +1,131 @@ +from realenv.data.datasets import ViewDataSet3D +from realenv.core.render.show_3d2 import PCRenderer, sync_coords +from realenv.core.channels.depth_render import run_depth_render +from realenv.core.physics.render_physics import PhysRenderer +from realenv import error + +import progressbar +import subprocess, os, signal +import numpy as np +import sys +import zmq +import socket +import shlex +from realenv.data.datasets import get_model_path + + +class Engine(object): + def __init__(self, model_id, human, debug): + self.dataset = ViewDataSet3D(transform = np.array, mist_transform = np.array, seqlen = 2, off_3d = False, train = False) + self.model_id = model_id + self.scale_up = 1 + self.human = human + self.debug = debug + self.r_visuals = None + self.r_physics = None + self.p_channel = None + + def setup_all(self): + def channel_excepthook(exctype, value, tb): + #if self.p_channel.is_alive(): + print("killing", self.p_channel) + #os.system("kill -9 {}".format(self.p_channel)) + self.p_channel.terminate() + while tb: + filename = tb.tb_frame.f_code.co_filename + name = tb.tb_frame.f_code.co_name + lineno = tb.tb_lineno + print ' File "%.500s", line %d, in %.500s' %(filename, lineno, name) + tb = tb.tb_next + print ' %s: %s' %(exctype.__name__, value) + sys.excepthook = channel_excepthook + + self._checkPortClear() + self._setupChannel() + self._setupPhysics(self.human) + self._setupVisuals() + + ## Sync initial poses + pose_init = self.r_visuals.renderOffScreenInitialPose() + self.r_physics = self._setupPhysics(self.human) + self.r_physics.initialize(pose_init) + + if self.debug: + self.r_visuals.renderToScreenSetup() + self.r_displayer = RewardDisplayer() #MPRewardDisplayer() + + return self.r_visuals, self.r_physics + + def _checkPortClear(self): + # TODO (hzyjerry) not working + """ + s = socket.socket() + try: + s.connect(("127.0.0.1", 5555)) + except socket.error as e: + raise e + raise error.Error("Realenv starting error: port {} is in use".format(5555)) + try: + s.connect(("127.0.0.1", 5556)) + except socket.error as e: + raise error.Error("Realenv starting error: port {} is in use".format(5556)) + """ + return + + def _setupChannel(self): + model_path = get_model_path() + dr_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'channels', 'depth_render') + cmd = "pwd && ls && cd {} && ./depth_render --modelpath {}".format(dr_path, model_path) + self.p_channel = subprocess.Popen(shlex.split(cmd), shell=False) + + + + def _setupVisuals(self): + scene_dict = dict(zip(self.dataset.scenes, range(len(self.dataset.scenes)))) + ## Todo: (hzyjerry) more error handling + if not self.model_id in scene_dict.keys(): + raise error.Error("Dataset not found: model {} cannot be loaded".format(self.model_id)) + else: + scene_id = scene_dict[self.model_id] + uuids, rts = self.dataset.get_scene_info(scene_id) + targets = [] + sources = [] + source_depths = [] + poses = [] + pbar = progressbar.ProgressBar(widgets=[ + ' [ Initializing Environment ] ', + progressbar.Bar(), + ' (', progressbar.ETA(), ') ', + ]) + for k,v in pbar(uuids): + data = self.dataset[v] + target = data[1] + target_depth = data[3] + + if self.scale_up !=1: + target = cv2.resize(target,None,fx=1.0/self.scale_up, fy=1.0/self.scale_up, interpolation = cv2.INTER_CUBIC) + target_depth = cv2.resize(target_depth,None,fx=1.0/self.scale_up, fy=1.0/self.scale_up, interpolation = cv2.INTER_CUBIC) + + pose = data[-1][0].numpy() + targets.append(target) + poses.append(pose) + sources.append(target) + source_depths.append(target_depth) + context_mist = zmq.Context() + socket_mist = context_mist.socket(zmq.REQ) + socket_mist.connect("tcp://localhost:5555") + + ## TODO (hzyjerry): make sure 5555&5556 are not occupied, or use configurable ports + sync_coords() + + renderer = PCRenderer(5556, sources, source_depths, target, rts, self.scale_up) + return renderer + + def _setupPhysics(self, human): + framePerSec = 13 + renderer = PhysRenderer(self.dataset.get_model_obj(), framePerSec, debug = self.debug, human = human) + return renderer + + def cleanUp(self): + self.p_channel.terminate() + \ No newline at end of file diff --git a/realenv/core/render/show_3d2.py b/realenv/core/render/show_3d2.py index a3f4f9bcf..9c7aa93c6 100644 --- a/realenv/core/render/show_3d2.py +++ b/realenv/core/render/show_3d2.py @@ -311,6 +311,12 @@ class PCRenderer: ## TODO (hzyjerry): does this introduce extra time delay? cv2.waitKey(1) return self.show_rgb + +def sync_coords(): + with Profiler("Transform coords"): + new_coords = np.getbuffer(coords.flatten().astype(np.uint32)) + socket_mist.send(new_coords) + message = socket_mist.recv() def show_target(target_img): @@ -319,18 +325,7 @@ def show_target(target_img): show_rgb = cv2.cvtColor(target_img, cv2.COLOR_BGR2RGB) cv2.imshow('target', show_rgb) -def sync_coords(): - print(coords.flatten().dtype) - with Profiler("Transform coords"): - new_coords = np.getbuffer(coords.flatten().astype(np.uint32)) - print(coords.shape) - print("Count: ", coords.flatten().astype(np.uint32).size ) - print("elem [2,3,5]: ", coords[4][2][1] ) - socket_mist.send(new_coords) - print("Sent reordering") - message = socket_mist.recv() - print("received reordering reply") - + if __name__=='__main__': parser = argparse.ArgumentParser() diff --git a/realenv/core/test_engine.py b/realenv/core/test_engine.py new file mode 100644 index 000000000..e69de29bb diff --git a/realenv/core/tests/test_engine.py b/realenv/core/tests/test_engine.py new file mode 100644 index 000000000..8f11636cb --- /dev/null +++ b/realenv/core/tests/test_engine.py @@ -0,0 +1,8 @@ +from realenv.core.engine import Engine +import time + +test_engine = Engine(model_id="11HB6XZSh1Q", human=True, debug=False) +test_engine.setup_all() + +#for i in range(3): +# time.sleep(1) diff --git a/realenv/envs/simple_env.py b/realenv/envs/simple_env.py index 1dd9ea603..6744e94f2 100644 --- a/realenv/envs/simple_env.py +++ b/realenv/envs/simple_env.py @@ -3,19 +3,14 @@ from gym import error, spaces, utils from gym.utils import seeding import realenv from realenv.main import RealEnv -from realenv.data.datasets import ViewDataSet3D -from realenv.core.render.show_3d2 import PCRenderer, sync_coords -from realenv.core.physics.render_physics import PhysRenderer +from realenv.core.engine import Engine from realenv.core.render.profiler import Profiler -from realenv.core.channels.depth_render import run_depth_render from realenv.core.scoreboard.realtime_plot import MPRewardDisplayer, RewardDisplayer import numpy as np import zmq import time import os import random -import progressbar -from multiprocessing import Process import cv2 @@ -26,71 +21,14 @@ class SimpleEnv(RealEnv): self.debug_mode = debug file_dir = os.path.dirname(__file__) - self.model_id = model_id - self.dataset = ViewDataSet3D(transform = np.array, mist_transform = np.array, seqlen = 2, off_3d = False, train = False) - - self.p_channel = Process(target=run_depth_render) + self.model_id = model_id self.state_old = None - self.scale_up = scale_up + self.scale_up = scale_up + self.engine = Engine(model_id, human, debug) - try: - self.p_channel.start() - self.r_visuals = self._setupVisuals() - pose_init = self.r_visuals.renderOffScreenInitialPose() - print("initial pose", pose_init) - self.r_physics = self._setupPhysics(human) - self.r_physics.initialize(pose_init) - if self.debug_mode: - self.r_visuals.renderToScreenSetup() - self.r_displayer = RewardDisplayer() #MPRewardDisplayer() - except Exception as e: - self._end() - raise(e) - - def _setupVisuals(self): - scene_dict = dict(zip(self.dataset.scenes, range(len(self.dataset.scenes)))) - if not self.model_id in scene_dict.keys(): - print("model not found") - else: - scene_id = scene_dict[self.model_id] - uuids, rts = self.dataset.get_scene_info(scene_id) - targets = [] - sources = [] - source_depths = [] - poses = [] - pbar = progressbar.ProgressBar(widgets=[ - ' [ Initializing Environment ] ', - progressbar.Bar(), - ' (', progressbar.ETA(), ') ', - ]) - for k,v in pbar(uuids): - data = self.dataset[v] - target = data[1] - target_depth = data[3] - - if self.scale_up !=1: - target = cv2.resize(target,None,fx=1.0/self.scale_up, fy=1.0/self.scale_up, interpolation = cv2.INTER_CUBIC) - target_depth = cv2.resize(target_depth,None,fx=1.0/self.scale_up, fy=1.0/self.scale_up, interpolation = cv2.INTER_CUBIC) - - pose = data[-1][0].numpy() - targets.append(target) - poses.append(pose) - sources.append(target) - source_depths.append(target_depth) - context_mist = zmq.Context() - socket_mist = context_mist.socket(zmq.REQ) - socket_mist.connect("tcp://localhost:5555") - - sync_coords() - - renderer = PCRenderer(5556, sources, source_depths, target, rts, self.scale_up) - return renderer + self.r_visuals, self.r_physics, self.p_channel = self.engine.setup_all() - def _setupPhysics(self, human): - framePerSec = 13 - renderer = PhysRenderer(self.dataset.get_model_obj(), framePerSec, debug = self.debug_mode, human = human) - return renderer def testShow3D(self): return @@ -128,11 +66,11 @@ class SimpleEnv(RealEnv): def _render(self, mode='human', close=False): return - - def _end(self): + + def __del__(self): ## TODO (hzyjerry): this does not kill cleanly ## to reproduce bug, set human = false, debug_mode = false - self.p_channel.terminate() + self.engine.cleanUp() return diff --git a/realenv/error.py b/realenv/error.py new file mode 100644 index 000000000..2a72b2dfb --- /dev/null +++ b/realenv/error.py @@ -0,0 +1,140 @@ +import sys + +class Error(Exception): + pass + +# Local errors + +class Unregistered(Error): + """Raised when the user requests an item from the registry that does + not actually exist. + """ + pass + +class UnregisteredEnv(Unregistered): + """Raised when the user requests an env from the registry that does + not actually exist. + """ + pass + +class UnregisteredBenchmark(Unregistered): + """Raised when the user requests an env from the registry that does + not actually exist. + """ + pass + +class DeprecatedEnv(Error): + """Raised when the user requests an env from the registry with an + older version number than the latest env with the same name. + """ + pass + +class UnseedableEnv(Error): + """Raised when the user tries to seed an env that does not support + seeding. + """ + pass + +class DependencyNotInstalled(Error): + pass + +class UnsupportedMode(Exception): + """Raised when the user requests a rendering mode not supported by the + environment. + """ + pass + +class ResetNeeded(Exception): + """When the monitor is active, raised when the user tries to step an + environment that's already done. + """ + pass + +class ResetNotAllowed(Exception): + """When the monitor is active, raised when the user tries to step an + environment that's not yet done. + """ + pass + +class InvalidAction(Exception): + """Raised when the user performs an action not contained within the + action space + """ + pass + +# API errors + +class APIError(Error): + def __init__(self, message=None, http_body=None, http_status=None, + json_body=None, headers=None): + super(APIError, self).__init__(message) + + if http_body and hasattr(http_body, 'decode'): + try: + http_body = http_body.decode('utf-8') + except: + http_body = ('') + + self._message = message + self.http_body = http_body + self.http_status = http_status + self.json_body = json_body + self.headers = headers or {} + self.request_id = self.headers.get('request-id', None) + + def __unicode__(self): + if self.request_id is not None: + msg = self._message or "" + return u"Request {0}: {1}".format(self.request_id, msg) + else: + return self._message + + if sys.version_info > (3, 0): + def __str__(self): + return self.__unicode__() + else: + def __str__(self): + return unicode(self).encode('utf-8') + + +class APIConnectionError(APIError): + pass + + +class InvalidRequestError(APIError): + + def __init__(self, message, param, http_body=None, + http_status=None, json_body=None, headers=None): + super(InvalidRequestError, self).__init__( + message, http_body, http_status, json_body, + headers) + self.param = param + + +class AuthenticationError(APIError): + pass + +class RateLimitError(APIError): + pass + +# Video errors + +class VideoRecorderError(Error): + pass + +class InvalidFrame(Error): + pass + +# Wrapper errors + +class DoubleWrapperError(Error): + pass + + +class WrapAfterConfigureError(Error): + pass + + +class RetriesExceededError(Error): + pass