diff --git a/README.md b/README.md index 6c1c8d354..32b0bf891 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Gibson Environment for Training Real World AI You shouldn't play video games all day, so shouldn't your AI. In this project we build a virtual environment that offers real world experience. You can think of it like [The Matrix](https://www.youtube.com/watch?v=3Ep_rnYweaI). +![gibson](misc/ui.gif) + + ### Note This is a 0.1.0 beta release, bug reports are welcome. @@ -103,29 +106,4 @@ More examples can be found in `examples/demo` and `examples/train` folder. Environment Configuration ================= -Each environment is configured with a `yaml` file. Examples of `yaml` files can be found in `examples/configs` folder. Parameters for the file is explained as below (take navigation environment for example): - -``` -envname: AntClimbEnv # Environment name, make sure it is the same as the class name of the environment -model_id: sRj553CTHiw # Scene id -target_orn: [0, 0, 3.14] # target orientation for navigating, the reference frame is world frame -target_pos: [-7, 2.6, -1.5] # target position for navigating, the reference frame is world frame -initial_orn: [0, 0, 3.14] # initial orientation for navigating -initial_pos: [-7, 2.6, 0.5] # initial position for navigating -fov: 1.57 # field of view for the camera -use_filler: true # use neural network filler or not, it is recommended to leave this argument true -display_ui: true # show pygame ui or not, if in a production environment (training), you need to turn this off -show_dignostic: true # show dignostics overlaying on the RGB image -ui_num: 2 # how many ui components to show -ui_components: [RGB_FILLED, DEPTH] # which are the ui components, choose from [RGB_FILLED, DEPTH, NORMAL, SEMANTICS, RGB_PREFILLED] - -output: [nonviz_sensor, rgb_filled, depth] # output of the environment to the robot -resolution: 512 # resolution of rgb/depth image - -speed: - timestep: 0.01 # timestep of simulation in seconds - frameskip: 1 # how many frames to run simulation for one action - -mode: gui # gui|headless, if in a production environment (training), you need to turn this to headless -verbose: false # show dignostics in terminal -``` +Each environment is configured with a `yaml` file. Examples of `yaml` files can be found in `examples/configs` folder. Parameters for the file is explained [here](gibson/configs/README.md). \ No newline at end of file diff --git a/examples/configs/README.md b/examples/configs/README.md new file mode 100644 index 000000000..c795f4351 --- /dev/null +++ b/examples/configs/README.md @@ -0,0 +1,28 @@ +Environment Configuration +================= +Each environment is configured with a `yaml` file. Examples of `yaml` files can be found in `examples/configs` folder. Parameters for the file is explained as below (take navigation environment for example): + +``` +envname: AntClimbEnv # Environment name, make sure it is the same as the class name of the environment +model_id: sRj553CTHiw # Scene id +target_orn: [0, 0, 3.14] # target orientation for navigating, the reference frame is world frame +target_pos: [-7, 2.6, -1.5] # target position for navigating, the reference frame is world frame +initial_orn: [0, 0, 3.14] # initial orientation for navigating +initial_pos: [-7, 2.6, 0.5] # initial position for navigating +fov: 1.57 # field of view for the camera +use_filler: true # use neural network filler or not, it is recommended to leave this argument true +display_ui: true # show pygame ui or not, if in a production environment (training), you need to turn this off +show_dignostic: true # show dignostics overlaying on the RGB image +ui_num: 2 # how many ui components to show +ui_components: [RGB_FILLED, DEPTH] # which are the ui components, choose from [RGB_FILLED, DEPTH, NORMAL, SEMANTICS, RGB_PREFILLED] + +output: [nonviz_sensor, rgb_filled, depth] # output of the environment to the robot +resolution: 512 # resolution of rgb/depth image + +speed: + timestep: 0.01 # timestep of simulation in seconds + frameskip: 1 # how many frames to run simulation for one action + +mode: gui # gui|headless, if in a production environment (training), you need to turn this to headless +verbose: false # show dignostics in terminal +``` diff --git a/gibson/README.md b/gibson/README.md deleted file mode 100644 index 65e4f47de..000000000 --- a/gibson/README.md +++ /dev/null @@ -1,21 +0,0 @@ -## Gibson Core Environment - -This folder contains the code base for running universe engine, as well as instructions needed for implementing custom environment and agent -### File Structure - - Client: client side code for running remote environment - - Core: realenv engine - - Data: realenv dataset - - Envs: repository of current environments, each can be thought of as a "challenge" - - Spaces: Envs dependencies - -### Implementing Agent - -```python -observation, reward, done, info = env._step({}) -``` - - - *observation* (object): agent's observation of the current environment - - *reward* (float) : amount of reward returned after previous action - - *done* (boolean): whether the episode has ended. The agent is responsible for taking care of this, by calling `env.restart()` - - *info* (dict): auxiliary diagnostic information (helpful for debugging, and sometimes learning) - diff --git a/gibson/core/Dockerfile b/gibson/core/Dockerfile deleted file mode 100644 index a7293082e..000000000 --- a/gibson/core/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -## Scratch ubuntu:16.04 image with NVIDIA GPU -FROM nvidia/cuda - -## Skip keyboard settings -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update \ - && apt-get install -y libav-tools \ - libpq-dev \ - libjpeg-dev \ - cmake \ - wget \ - unzip \ - git \ - xpra \ - golang-go \ - libboost-all-dev \ - make \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - -## Install conda, opencv - -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.14-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh - -ENV PATH /opt/conda/bin:$PATH - -RUN conda install -c menpo opencv -y -#RUN conda install -y \ -# scikit-image \ -# flask \ -# pillow - -RUN conda install pytorch torchvision cuda80 -c soumith - -## Install Universe -WORKDIR /usr/local/realenv -RUN git clone https://github.com/openai/universe.git -WORKDIR /usr/local/realenv/universe -RUN pip install -e . - -## Install Realenv -WORKDIR /usr/local/realenv -RUN wget https://www.dropbox.com/s/xmhgkmhgp9dfw52/realenv.tar.gz && tar -xvzf realenv.tar.gz && rm realenv.tar.gz -WORKDIR /usr/local/realenv/realenv -RUN pip install -e . - -## Set up data & model for view synthesizer -#nvidia-docker run -it -v realenv-data:/usr/local/realenv/data realenv bash - -## Start VNC server - - - -## Entry point -WORKDIR /usr/local/realenv/ -ENTRYPOINT [ "/bin/bash", "-c" ] - diff --git a/gibson/core/README.md b/gibson/core/README.md deleted file mode 100644 index 38a3d9dfe..000000000 --- a/gibson/core/README.md +++ /dev/null @@ -1,66 +0,0 @@ -## Real Universe Environment - -This folder has the source code for environment bundle, which includes view renderer, multichannel renderer and physics simulation engine - -### Requirement -You need CUDA 8.0 in order to run the environment. - -### Run everything in a bundle -Current trainin environment in implemented in `simple_env.py`. It is a dead simple room exploration game with negative delta distance as reward. It's default to ru with model `11HB6XZSh1Q`. We provide a sample agent `random_husky.py` to interact with this environment. To start, run the following command -```shell -source activate (py2_universe_env) -python random_husky.py -``` -You can switch between agent/human mode by setting `human=True/False` when initializing environment object. You can also set `debug=True/False` to run with or without graphical interface. - -### Components -The above environment has three components: view renderer, multichannel renderer and physics simulation engine. A built-in add on is a scoreboard implemented with matplotlib. - -#### Test each component individually -Multichannel renderer currently only supports depth rendering. In the future we will provide surface normal, semantics and arbitrary meta information. -```shell -## Shell One -cd depth/depth_render -./depth_render --datapath ../../data -m 11HB6XZSh1Q -``` - -View renderer. -```shell -## Shell Two -cd render -source activate (py2_universe_env) -python show_3d2.py --datapath ../data/ --idx 10 -``` - -Physics engine -```shell -## Shell Three -cd physics -source activate (py2_universe_env) -python render_physics.py --datapath ../data --model 11HB6XZSh1Q -``` - -If you want to test out how they work together in a combined way, you don't have to open up three separate terminals. Run -```shell -source acitvate (py2_universe_env) -python simple_env.py -``` - -You should see OpenCV windows, as well as pybullet panel pop up. - -To control the object, click on pybullet panel, then use the following keys. Pybullet sends its movement to -Opengl & OpenCV for rendering. - -| key | action | -| ------------- | ------------- | -| w/q | go forward | -| a | go left | -| s | go backward | -| d | go right | -| z | go upward | -| c | go downward | -| u/j | add/subtract roll | -| i/k | add/subtract pitch | -| o/l | add/subtract yaw | - -Note that key w has conflict with pybullet hotkey. \ No newline at end of file diff --git a/gibson/core/render/README.md b/gibson/core/render/README.md deleted file mode 100644 index ff4e292ea..000000000 --- a/gibson/core/render/README.md +++ /dev/null @@ -1,30 +0,0 @@ -## Real Universe Viewer Renderer - -Renderer for real-time visual input. Currently neural network filler is not included, rendering framerate: 30fps in single view, 12fps with five views. - -Beta version renderer is implemented using OpenCV-Python. In the future we might completely switch to OpenGL to avoid the speed overhead, and reduce setup cost. - -Install opencv, pytorch and others -```shell -conda install -c menpo opencv -y -conda install pytorch torchvision cuda80 -c soumith -conda install pillow -pip install zmq -``` - -Build cpp, cython dependencies -```shell -bash build.sh -bash build_cuda.sh -python setup.py build_ext --inplace -``` - -Download the necessary helper files -```shell -wget https://www.dropbox.com/s/a4tb959bbue0hap/coord.npy -``` - -``` -source activate (py2_universe_env) -python show_3d2.py --datapath ../data/ --idx 10 -``` \ No newline at end of file diff --git a/gibson/core/scoreboard/__init__.py b/gibson/core/scoreboard/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gibson/core/scoreboard/realtime_plot.py b/gibson/core/scoreboard/realtime_plot.py deleted file mode 100644 index d0bfa1102..000000000 --- a/gibson/core/scoreboard/realtime_plot.py +++ /dev/null @@ -1,198 +0,0 @@ -import time, random -import math -import collections -from collections import deque -from matplotlib.font_manager import FontProperties -import matplotlib.gridspec as gridspec -import matplotlib.patches as mpatches -from matplotlib import pyplot as plt -from multiprocessing import Process, Pipe -#import gobject - -class SmoothList(collections.MutableSequence): - max_entries = 200 - def __init__(self): - self.list = [] - - def __getitem__(self, idx): - scale = 1 - if len(self.list) > SmoothList.max_entries: - scale = len(self.list) / float(SmoothList.max_entries) - return self.list[int((idx + 1) * scale) - 1] - - def __len__(self): - return min(len(self.list), SmoothList.max_entries) - - def append(self, elem): - ''' - if len(self.list) >= SmoothList.max_entries: - newlist = [] - i = 0 - while i < len(self.list): - newlist.append((self.list[i] + self.list[i + 1]) / 2.0) - i = i + 2 - self.list = newlist - ''' - self.list.append(elem) - - def __delitem__(self, i): del self.list[i] - - def __setitem__(self, i, v): - self.check(v) - self.list[i] = v - - def insert(self, i, v): - self.check(v) - self.list.insert(i, v) - - -class RealtimePlot: - def __init__(self, axes, max_entries = 100, full = False): - self.axes = axes - self.max_entries = max_entries - if not full: - self.axis_x = deque(maxlen=max_entries) - self.axis_y = deque(maxlen=max_entries) - self.lineplot, = axes.plot([], [], "bo-") - else: - self.axis_x = SmoothList() - self.axis_y = SmoothList() - self.lineplot, = axes.plot([], [], "ro-") - - self.axes.set_autoscaley_on(True) - - def add(self, x, y): - self.axis_x.append(x) - self.axis_y.append(y) - self.lineplot.set_data(self.axis_x, self.axis_y) - self.axes.set_xlim(self.axis_x[0], self.axis_x[-1] + 1e-15) - self.axes.relim(); self.axes.autoscale_view() # rescale the y-axis - - def animate(self, figure, callback, interval = 50): - import matplotlib.animation as animation - def wrapper(frame_index): - self.add(*callback(frame_index)) - self.axes.relim(); self.axes.autoscale_view() # rescale the y-axis - return self.lineplot - animation.FuncAnimation(figure, wrapper, interval=interval) - - -class RewardDisplayer: - def __init__(self): - self.all_rewards = [] - self.factor = 3 - font0 = FontProperties() - gs = gridspec.GridSpec(12, 3) - - fig = plt.figure(figsize=(10.25, 8)) - - self.axes_text = plt.subplot(gs[:2, :], facecolor='gainsboro') - self.axes_full = plt.subplot(gs[2:9, :], facecolor='gainsboro') - self.axes_real = plt.subplot(gs[9:12, :], facecolor='gainsboro') - - self.axes_text.set_xlim(0, 10) - self.axes_text.set_ylim(0, 10) - - #ax1.set_title('title1', color='c', rotation='vertical',x=-0.1,y=0.5) - self.axes_full.legend(loc='upper left', handles=[mpatches.Patch(color='red', label='Score - Time Plot')], prop={'size': 13}) - self.axes_real.legend(loc='upper left', handles=[mpatches.Patch(color='blue', label='Reward - Time Plot')], prop={'size': 13}) - - self.font = font0.copy() - self.font.set_family('sans caption') - self.font.set_weight('semibold') - self.alignment = {'horizontalalignment': 'left', 'verticalalignment': 'baseline'} - self.axes_text.axis('off') - - self.display_full = RealtimePlot(self.axes_full, full = True) - self.display_real = RealtimePlot(self.axes_real) - - self.txt_reward = None - self.txt_totalr = None - self.txt_time = None - - self.axes_text.text(0, 9, 'Reward:', fontproperties=self.font, size = 15, **self.alignment) - self.axes_text.text(4.5, 9, 'Score:', fontproperties=self.font, size = 15, **self.alignment) - self.axes_text.text(0, 3, 'Time:', fontproperties=self.font, size = 15, **self.alignment) - self.score = 0 - - self.start = time.time() - - def _smooth(self): - return self.all_rewards - - def reset(self): - self.score = 0 - - def add_reward(self, reward): - self.score = self.score + reward - cur_time = time.time() - self.start - self.display_real.add(cur_time, reward) - self.display_full.add(cur_time, self.score) - if self.txt_reward: - self.txt_reward.remove() - if self.txt_totalr: - self.txt_totalr.remove() - if self.txt_time: - self.txt_time.remove() - self.txt_reward = self.axes_text.text(2, 9, '%.2f' % reward, fontproperties=self.font, size = 30, **self.alignment) - self.txt_totalr = self.axes_text.text(6.5, 9, '%.2f' % self.score, fontproperties=self.font, size = 30, **self.alignment) - self.txt_time = self.axes_text.text(2, 3, '%.2f' % cur_time, fontproperties=self.font, size = 30, **self.alignment) - plt.pause(0.0001) - - def terminate(self): - plt.close('all') - - def poll_draw(self): - while 1: - #print("polling") - if not self.pipe.poll(): - break - command = self.pipe.recv() - #print("received reward", command) - if command is None: - self.terminate() - return False - else: - self.add_reward(command) - time.sleep(0.01) - #self.fig.canvas.draw() - return True - - def __call__(self, pipe): - print('Starting plotter...') - - self.pipe = pipe - #self.fig, self.ax = plt.subplots() - #self.gid = gobject.timeout_add(1000, ) - self.poll_draw() - print('...done') - #plt.show() - -class MPRewardDisplayer(object): - def __init__(self): - self.plot_pipe, plotter_pipe = Pipe() - self.plotter = RewardDisplayer() - self.plot_process = Process(target=self.plotter, - args=(plotter_pipe,)) - self.plot_process.daemon = True - self.plot_process.start() - - def add_reward(self, reward, finished=False): - send = self.plot_pipe.send - if finished: - send(None) - else: - send(reward) - - def reset(self): - return - -def main(): - r_displayer = MPRewardDisplayer() - for i in range(10000): - num = random.random() * 100 - 30 - r_displayer.add_reward(num) - if i % 40 == 0: - r_displayer.reset() - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/misc/ui.gif b/misc/ui.gif new file mode 100644 index 000000000..1ed187737 Binary files /dev/null and b/misc/ui.gif differ