Some workflow, CI and release improvements (#107)

* Start preparing CI for release

* Change docker omnigibson dirname to avoid path conflict during import

* Create protect-main.yml

* Bolder comment.

* Fix branch name

* Split dockerfile into dev/prod use cases

* Build docker-based test and docs workflow

* Try with source instead of bash

* Force docs rm

.github/workflows/docs.yml

@@ -1,32 +1,62 @@
 name: Build & deploy docs
 
-on: [push]
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
 
 jobs:
-  build-docs:
+  docs:
-    name: Build and deploy documentation
+    runs-on: [linux]
-    runs-on: ubuntu-latest
+    container:
-    if: github.repository == 'StanfordVL/OmniGibson' && github.ref == 'refs/heads/master'
+      image: stanfordvl/omnigibson-dev:latest
+      options: --gpus=all --privileged --user=root
+      env:
+        DISPLAY: ""
+        OMNIGIBSON_HEADLESS: 1
+      volumes:
+        - /scr/omni-data/datasets:/data
+        - /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json
+        - /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json
+        - /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json
+        - /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw
+        - /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw
+        - /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
+        - /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
+        - /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
+        - /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw
+        - /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw
+        - /scr/omni-data/isaac-sim/documents:/root/Documents:rw
+
+    defaults:
+      run:
+        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
 
     steps:
-    - name: Checkout code
+      - name: Fix home
-      uses: actions/checkout@v2
+        run: echo "HOME=/root" >> $GITHUB_ENV
-    
-    - name: Setup python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.8"
-        architecture: x64
 
-    - name: Install dev requirements
+      - name: Checkout source
-      run: pip install -r requirements-dev.txt
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+          path: omnigibson-src
 
-    - name: Generate docs
+      - name: Install dev requirements
-      working-directory: ./docs
+        working-directory: omnigibson-src
-      run: make html
+        run: pip install -r requirements-dev.txt
 
-    - name: Deploy to gh-pages
+      - name: Install
-      uses: peaceiris/actions-gh-pages@v3
+        working-directory: omnigibson-src
-      with:
+        run: pip install -e .
-        github_token: ${{ secrets.GITHUB_TOKEN }}
+
-        publish_dir: ./docs/_build/html
+      - name: Build docs
+        working-directory: omnigibson-src
+        run: source /isaac-sim/setup_conda_env.sh && source build_docs.sh
+
+      - name: Deploy to gh-pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./omnigibson-src/site

.github/workflows/protect-main.yml

@@ -0,0 +1,22 @@
+name: Disallow PRs to main branch
+
+on: [pull_request]
+
+jobs:
+  protect-main:
+    name: Disallow PRs to main branch
+    runs-on: ubuntu-latest
+    if: ${{ github.base_ref == 'main' }}
+
+    steps:
+    - name: Comment on PR
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        message: |
+          Hi, research friend! :wave:
+          It looks like you are trying to merge a PR to the `main` branch.
+          This branch is for released code only. **Please PR your code onto `og-develop` instead.**
+          Thank you!
+
+    - name: Fail
+      run: exit 1

.github/workflows/release.yml

@@ -1,40 +0,0 @@
-name: Release on PyPI
-
-on:
-  release:
-    types:
-      - created
-
-jobs:
-  build:
-    runs-on: "ubuntu-latest"
-    steps:
-    - name: Checkout OmniGibson source
-      uses: actions/checkout@master
-      with:
-        submodules: true
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Install pypa/build
-      run: >-
-        python -m
-        pip install
-        build
-        --user
-    - name: Remove unnecessary files to reduce file size
-      run: rm -r omnigibson/render/openvr/samples
-    - name: Build a binary wheel and a source tarball
-      run: >-
-        python -m
-        build
-        --sdist
-        --outdir dist/
-        .
-    - name: Publish a Python distribution to PyPI
-      if: github.repository == 'StanfordVL/OmniGibson' && startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/sync-repos.yml

@@ -1,18 +0,0 @@
-name: Sync OmniGibson-dev/master to OmniGibson/master
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Sync repositories on the 1st and 15th of the month at 9 AM
-    - cron:  "0 17 1,15 * *" 
-
-jobs:
-  sync-private-fork:
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master'
-    steps:
-      - uses: actions/checkout@v2
-      - name: Sync OmniGibson-dev/master to OmniGibson/master
-        if: github.repository == 'StanfordVL/OmniGibson-dev' && startsWith(github.ref, 'refs/tags')
-        run:
-          git push https://$PERSONAL_ACCESS_TOKEN:x-oauth-basic@github.com/StanfordVL/OmniGibson.git master:master

.github/workflows/tests.yml

@@ -9,81 +9,53 @@ concurrency:
 jobs:
   test:
     runs-on: [self-hosted, linux, gpu]
-    if: github.repository == 'StanfordVL/OmniGibson-dev'
+    container:
+      image: stanfordvl/omnigibson-dev:latest
+      options: --gpus=all --privileged --user=root
+      env:
+        DISPLAY: ""
+        OMNIGIBSON_HEADLESS: 1
+      volumes:
+        - /scr/omni-data/datasets:/data
+        - /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json
+        - /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json
+        - /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json
+        - /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw
+        - /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw
+        - /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
+        - /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
+        - /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
+        - /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw
+        - /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw
+        - /scr/omni-data/isaac-sim/documents:/root/Documents:rw
+
+    defaults:
+      run:
+        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
 
     steps:
+      - name: Fix home
+        run: echo "HOME=/root" >> $GITHUB_ENV
+
       - name: Checkout source
         uses: actions/checkout@v2
         with:
           submodules: true
-          path: omnigibson
+          path: omnigibson-src
-
-      - name: Add CUDA to env
-        run: echo "/usr/local/cuda/bin" >> $GITHUB_PATH
-
-      - name: Setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-          architecture: x64
 
       - name: Install dev requirements
-        working-directory: omnigibson
+        working-directory: omnigibson-src
         run: pip install -r requirements-dev.txt
 
       - name: Install
-        working-directory: omnigibson
+        working-directory: omnigibson-src
         run: pip install -e .
 
-      - name: Uninstall pip bddl
-        working-directory: omnigibson
-        run: pip uninstall -y bddl
-
-      - name: Checkout BDDL
-        uses: actions/checkout@v2
-        with:
-          repository: StanfordVL/bddl-dev
-          ref: 581be50e7cfd2b3a1447aaa1b4fc2424b673339c
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}  # PAT is required since this is a different repo
-          path: bddl
-          submodules: recursive
-          lfs: true
-
-      - name: Install BDDL
-        working-directory: bddl
-        run: pip install -e .
-
-      - name: Link Dataset
-        working-directory: omnigibson
-        run: ln -s /scr/ig-data omnigibson/data
-
-      # The below method of checking out ig-dataset is currently unused due to poor speed.
-      # - name: Create data directory
-      #   run: mkdir -p omnigibson/omnigibson/data
-      #
-      # - name: Checkout og_dataset
-      #   uses: actions/checkout@v2
-      #   with:
-      #     repository: StanfordVL/og_dataset
-      #     token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}  # PAT is required since this is a different repo
-      #     path: omnigibson/omnigibson/data/og_dataset
-      #     submodules: recursive
-      #     lfs: true
-      #
-      # - name: Checkout og_assets
-      #   uses: actions/checkout@v2
-      #   with:
-      #     repository: StanfordVL/og_assets
-      #     token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}  # PAT is required since this is a different repo
-      #     path: omnigibson/omnigibson/data/assets
-      #     submodules: recursive
-      #     lfs: true
-
       - name: Run tests
-        working-directory: omnigibson
+        working-directory: omnigibson-src
-        run: pytest
+        run: source /isaac-sim/setup_conda_env.sh && pytest
 
-      - name: Upload coverage to Codecov
+      # - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2.1.0
+      #   uses: codecov/codecov-action@v2.1.0
-        with:
+      #   with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+      #     token: ${{ secrets.CODECOV_TOKEN }}

build_docs.sh

@@ -1,17 +1,12 @@
 #!/usr/bin/env bash
 
-# Make sure you run this script EXACTLY as `bash ./build_docs.sh`
-
-# Activate conda env
-source activate omnigibson
-
 # Remove source directory and copy over source files to docs folder
-rm -r docs/src
+rm -rf docs/src
 mkdir docs/src
 cp -r omnigibson/* docs/src
 
 # Update code source references
-rm -r docs/reference
+rm -rf docs/reference
 python docs/gen_ref_pages.py
 
 # Build the docs (written to ./site)

docker/build_docker.sh

@@ -1,7 +1,12 @@
 #!/usr/bin/env bash
 
+docker build \
+    -t stanfordvl/omnigibson-dev:latest \
+    -f docker/dev.Dockerfile \
+    .
+
 docker build \
     -t stanfordvl/omnigibson:latest \
     -t stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py) \
-    -f docker/Dockerfile \
+    -f docker/prod.Dockerfile \
     .

docker/dev.Dockerfile

@@ -10,10 +10,6 @@ RUN apt-get update && apt-get install -y \
 RUN rm -rf /isaac-sim/exts/omni.isaac.ml_archive/pip_prebundle/gym
 RUN rm -rf /isaac-sim/kit/extscore/omni.kit.pip_archive/pip_prebundle/numpy
 
-# Add OmniGibson
-ADD . /omnigibson
-WORKDIR /omnigibson
-
 # Mount the data directory
 VOLUME ["/data"]
 ENV OMNIGIBSON_DATASET_PATH /data/og_dataset
@@ -33,10 +29,6 @@ RUN echo "source /isaac-sim/setup_conda_env.sh" >> /root/.bashrc
 SHELL ["micromamba", "run", "-n", "omnigibson", "/bin/bash", "--login", "-c"]
 RUN micromamba install -c conda-forge opencv
 
-# Install OmniGibson
-RUN pip install -e .
-
-RUN echo "OMNIGIBSON_NO_OMNIVERSE=1 python -m omnigibson.scripts.setup" >> /root/.bashrc
 ENTRYPOINT []
 
 CMD ["/bin/bash"]

docker/prod.Dockerfile

@@ -0,0 +1,14 @@
+FROM stanfordvl/omnigibson-dev
+
+# Copy over omnigibson source
+ADD . /omnigibson-src
+WORKDIR /omnigibson-src
+
+# Install OmniGibson
+RUN pip install -e .
+
+# Add setup to be executed on bash launch
+RUN echo "OMNIGIBSON_NO_OMNIVERSE=1 python -m omnigibson.scripts.setup" >> /root/.bashrc
+ENTRYPOINT []
+
+CMD ["/bin/bash"]

docker/push_docker.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 
 docker push stanfordvl/omnigibson:latest
-docker push stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py)
+docker push stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py)
+docker push stanfordvl/omnigibson-dev:latest

requirements-dev.txt

@@ -0,0 +1,10 @@
+pytest>=6.2.3
+pytest-cov>=3.0.0
+mkdocs
+mkdocs-autorefs
+mkdocs-gen-files
+mkdocs-material
+mkdocs-material-extensions
+mkdocstrings[python]
+mkdocs-section-index
+mkdocs-literate-nav