p04798526
/
OmniGibson
mirror of https://github.com/StanfordVL/OmniGibson.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some workflow, CI and release improvements (#107) 

* Start preparing CI for release

* Change docker omnigibson dirname to avoid path conflict during import

* Create protect-main.yml

* Bolder comment.

* Fix branch name

* Split dockerfile into dev/prod use cases

* Build docker-based test and docs workflow

* Try with source instead of bash

* Force docs rm
This commit is contained in:
Cem Gökmen 2023-04-05 17:25:26 -07:00 committed by GitHub
parent add5a58da8
commit f485f6e77a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 144 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
.github/workflows/docs.yml vendored
View File

@ -1,32 +1,62 @@
name: Build & deploy docs
on: [push]
on: [pull_request]
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
jobs:
build-docs:
name: Build and deploy documentation
runs-on: ubuntu-latest
if: github.repository == 'StanfordVL/OmniGibson' && github.ref == 'refs/heads/master'
docs:
runs-on: [linux]
container:
image: stanfordvl/omnigibson-dev:latest
options: --gpus=all --privileged --user=root
env:
DISPLAY: ""
OMNIGIBSON_HEADLESS: 1
volumes:
- /scr/omni-data/datasets:/data
- /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json
- /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json
- /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json
- /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw
- /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw
- /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
- /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
- /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
- /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw
- /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw
- /scr/omni-data/isaac-sim/documents:/root/Documents:rw
defaults:
run:
shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup python
uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: x64
- name: Fix home
run: echo "HOME=/root" >> $GITHUB_ENV
- name: Install dev requirements
run: pip install -r requirements-dev.txt
- name: Checkout source
uses: actions/checkout@v2
with:
submodules: true
path: omnigibson-src
- name: Generate docs
working-directory: ./docs
run: make html
- name: Install dev requirements
working-directory: omnigibson-src
run: pip install -r requirements-dev.txt
- name: Deploy to gh-pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs/_build/html
- name: Install
working-directory: omnigibson-src
run: pip install -e .
- name: Build docs
working-directory: omnigibson-src
run: source /isaac-sim/setup_conda_env.sh && source build_docs.sh
- name: Deploy to gh-pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./omnigibson-src/site

22
.github/workflows/protect-main.yml vendored Normal file
View File

@ -0,0 +1,22 @@
name: Disallow PRs to main branch
on: [pull_request]
jobs:
protect-main:
name: Disallow PRs to main branch
runs-on: ubuntu-latest
if: ${{ github.base_ref == 'main' }}
steps:
- name: Comment on PR
uses: thollander/actions-comment-pull-request@v2
with:
message: |
Hi, research friend! :wave:
It looks like you are trying to merge a PR to the `main` branch.
This branch is for released code only. **Please PR your code onto `og-develop` instead.**
Thank you!
- name: Fail
run: exit 1

40
.github/workflows/release.yml vendored
View File

@ -1,40 +0,0 @@
name: Release on PyPI
on:
release:
types:
- created
jobs:
build:
runs-on: "ubuntu-latest"
steps:
- name: Checkout OmniGibson source
uses: actions/checkout@master
with:
submodules: true
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install pypa/build
run: >-
python -m
pip install
build
--user
- name: Remove unnecessary files to reduce file size
run: rm -r omnigibson/render/openvr/samples
- name: Build a binary wheel and a source tarball
run: >-
python -m
build
--sdist
--outdir dist/
.
- name: Publish a Python distribution to PyPI
if: github.repository == 'StanfordVL/OmniGibson' && startsWith(github.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}

18
.github/workflows/sync-repos.yml vendored
View File

@ -1,18 +0,0 @@
name: Sync OmniGibson-dev/master to OmniGibson/master
on:
workflow_dispatch:
schedule:
# Sync repositories on the 1st and 15th of the month at 9 AM
- cron: "0 17 1,15 * *"
jobs:
sync-private-fork:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/master'
steps:
- uses: actions/checkout@v2
- name: Sync OmniGibson-dev/master to OmniGibson/master
if: github.repository == 'StanfordVL/OmniGibson-dev' && startsWith(github.ref, 'refs/tags')
run:
git push https://$PERSONAL_ACCESS_TOKEN:x-oauth-basic@github.com/StanfordVL/OmniGibson.git master:master

98
.github/workflows/tests.yml vendored
View File

@ -9,81 +9,53 @@ concurrency:
jobs:
test:
runs-on: [self-hosted, linux, gpu]
if: github.repository == 'StanfordVL/OmniGibson-dev'
container:
image: stanfordvl/omnigibson-dev:latest
options: --gpus=all --privileged --user=root
env:
DISPLAY: ""
OMNIGIBSON_HEADLESS: 1
volumes:
- /scr/omni-data/datasets:/data
- /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json
- /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json
- /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json
- /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw
- /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw
- /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
- /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
- /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
- /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw
- /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw
- /scr/omni-data/isaac-sim/documents:/root/Documents:rw
defaults:
run:
shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
steps:
- name: Fix home
run: echo "HOME=/root" >> $GITHUB_ENV
- name: Checkout source
uses: actions/checkout@v2
with:
submodules: true
path: omnigibson
- name: Add CUDA to env
run: echo "/usr/local/cuda/bin" >> $GITHUB_PATH
- name: Setup python
uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: x64
path: omnigibson-src
- name: Install dev requirements
working-directory: omnigibson
working-directory: omnigibson-src
run: pip install -r requirements-dev.txt
- name: Install
working-directory: omnigibson
working-directory: omnigibson-src
run: pip install -e .
- name: Uninstall pip bddl
working-directory: omnigibson
run: pip uninstall -y bddl
- name: Checkout BDDL
uses: actions/checkout@v2
with:
repository: StanfordVL/bddl-dev
ref: 581be50e7cfd2b3a1447aaa1b4fc2424b673339c
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT is required since this is a different repo
path: bddl
submodules: recursive
lfs: true
- name: Install BDDL
working-directory: bddl
run: pip install -e .
- name: Link Dataset
working-directory: omnigibson
run: ln -s /scr/ig-data omnigibson/data
# The below method of checking out ig-dataset is currently unused due to poor speed.
# - name: Create data directory
# run: mkdir -p omnigibson/omnigibson/data
#
# - name: Checkout og_dataset
# uses: actions/checkout@v2
# with:
# repository: StanfordVL/og_dataset
# token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT is required since this is a different repo
# path: omnigibson/omnigibson/data/og_dataset
# submodules: recursive
# lfs: true
#
# - name: Checkout og_assets
# uses: actions/checkout@v2
# with:
# repository: StanfordVL/og_assets
# token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT is required since this is a different repo
# path: omnigibson/omnigibson/data/assets
# submodules: recursive
# lfs: true
- name: Run tests
working-directory: omnigibson
run: pytest
working-directory: omnigibson-src
run: source /isaac-sim/setup_conda_env.sh && pytest
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2.1.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v2.1.0
# with:
# token: ${{ secrets.CODECOV_TOKEN }}

9
build_docs.sh
View File

@ -1,17 +1,12 @@
#!/usr/bin/env bash
# Make sure you run this script EXACTLY as `bash ./build_docs.sh`
# Activate conda env
source activate omnigibson
# Remove source directory and copy over source files to docs folder
rm -r docs/src
rm -rf docs/src
mkdir docs/src
cp -r omnigibson/* docs/src
# Update code source references
rm -r docs/reference
rm -rf docs/reference
python docs/gen_ref_pages.py
# Build the docs (written to ./site)

7
docker/build_docker.sh
View File

@ -1,7 +1,12 @@
#!/usr/bin/env bash
docker build \
-t stanfordvl/omnigibson-dev:latest \
-f docker/dev.Dockerfile \
.
docker build \
-t stanfordvl/omnigibson:latest \
-t stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py) \
-f docker/Dockerfile \
-f docker/prod.Dockerfile \
.

8
docker/Dockerfile → docker/dev.Dockerfile
View File

@ -10,10 +10,6 @@ RUN apt-get update && apt-get install -y \
RUN rm -rf /isaac-sim/exts/omni.isaac.ml_archive/pip_prebundle/gym
RUN rm -rf /isaac-sim/kit/extscore/omni.kit.pip_archive/pip_prebundle/numpy
# Add OmniGibson
ADD . /omnigibson
WORKDIR /omnigibson
# Mount the data directory
VOLUME ["/data"]
ENV OMNIGIBSON_DATASET_PATH /data/og_dataset
@ -33,10 +29,6 @@ RUN echo "source /isaac-sim/setup_conda_env.sh" >> /root/.bashrc
SHELL ["micromamba", "run", "-n", "omnigibson", "/bin/bash", "--login", "-c"]
RUN micromamba install -c conda-forge opencv
# Install OmniGibson
RUN pip install -e .
RUN echo "OMNIGIBSON_NO_OMNIVERSE=1 python -m omnigibson.scripts.setup" >> /root/.bashrc
ENTRYPOINT []
CMD ["/bin/bash"]

14
docker/prod.Dockerfile Normal file
View File

@ -0,0 +1,14 @@
FROM stanfordvl/omnigibson-dev
# Copy over omnigibson source
ADD . /omnigibson-src
WORKDIR /omnigibson-src
# Install OmniGibson
RUN pip install -e .
# Add setup to be executed on bash launch
RUN echo "OMNIGIBSON_NO_OMNIVERSE=1 python -m omnigibson.scripts.setup" >> /root/.bashrc
ENTRYPOINT []
CMD ["/bin/bash"]

3
docker/push_docker.sh
View File

@ -1,4 +1,5 @@
#!/usr/bin/env bash
docker push stanfordvl/omnigibson:latest
docker push stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py)
docker push stanfordvl/omnigibson:$(sed -ne "s/.*version= *['\"]\([^'\"]*\)['\"] *.*/\1/p" setup.py)
docker push stanfordvl/omnigibson-dev:latest

10
requirements-dev.txt Normal file
View File

@ -0,0 +1,10 @@
pytest>=6.2.3
pytest-cov>=3.0.0
mkdocs
mkdocs-autorefs
mkdocs-gen-files
mkdocs-material
mkdocs-material-extensions
mkdocstrings[python]
mkdocs-section-index
mkdocs-literate-nav