Merge pull request #375 from PTG-Kitware/experiment/upgrade-to-iron

Upgrade to ROS2 Iron

.github/workflows/ci.yml

@@ -31,17 +31,61 @@ jobs:
       # Use scripting to perform a build for the current state of this branch.
       - name: Build image
         run: ./angel-docker-build.sh
+      # Make sure workspace builds with this environment
+      - name: Build Workspace
+        run: ./angel-workspace-shell.sh -s workspace-shell-dev ./workspace_build.sh
 
   code-style-py:
     runs-on: ubuntu-latest
-    container: pyfound/black:latest_release
     steps:
       # Check out repository under $GITHUB_WORKSPACE so our job can access it.
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      # TODO: Make use of build docker image somehow below this point
+      # setup-python with poetry requires poetry to already be installed
+      - run: |
+          echo "${HOME}/.local/bin" >> $GITHUB_PATH
+          pip install --user -U poetry
+      # Setup python version and caching
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10.12'  # matching python version in container env.
+          cache: 'poetry'
+      # Install standard packages
+      - run: |
+          sudo apt install -y libasound2-dev  # for python simpleaudio
+          poetry install
+      # Run black
       - name: Lint python code
         run: |
-          black --version
-          black --check --diff --color .
+          poetry run black --version
+          poetry run black --check --diff --color .
+
+  type-checking:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      # Check out repository under $GITHUB_WORKSPACE so our job can access it.
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      # TODO: Make use of build docker image somehow below this point
+      # setup-python with poetry requires poetry to already be installed
+      - run: |
+          echo "${HOME}/.local/bin" >> $GITHUB_PATH
+          pip install --user -U poetry
+      # Setup python version and caching
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10.12'  # matching python version in container env.
+          cache: 'poetry'
+      # Install standard packages
+      - run: |
+          sudo apt install -y libasound2-dev  # for python simpleaudio
+          poetry install
+      # Run mypy to check types
+      - run: poetry run mypy
 
   pytest-angel-system:
     runs-on: ubuntu-latest
@@ -58,7 +102,7 @@ jobs:
       # Setup python version and caching
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'  # matching python version in container env.
+          python-version: '3.10.12'  # matching python version in container env.
           cache: 'poetry'
       # Install standard packages
       - run: |

.gitmodules

@@ -1,10 +1,6 @@
 [submodule "ros/ROS-TCP-Endpoint"]
 	path = ros/ROS-TCP-Endpoint
 	url = https://github.com/Unity-Technologies/ROS-TCP-Endpoint.git
-[submodule "ros/rosbag2"]
-	path = ros/rosbag2
-	url = https://github.com/ros2/rosbag2.git
-	branch = foxy-future
 [submodule "hl2ss"]
 	path = hl2ss
 	url = https://github.com/VIDA-NYU/hl2ss.git

.mypy.ini

@@ -0,0 +1,26 @@
+###############################################################################
+# Global Options
+
+[mypy]
+# Files/Directories to investigate.
+files = angel_system, tests, ros/angel_utils/python/angel_utils, ros/angel_system_nodes/angel_system_nodes
+
+# Disable caching
+incremental = False
+
+# Disallows defining functions without type annotations or with incomplete type
+# annotations.
+disallow_untyped_defs = True
+
+###############################################################################
+# Options for different parts of angel_system
+
+;[mypy-angel_system.*]
+
+;[mypy-tests.*]
+
+###############################################################################
+# TPL-module options
+
+[mypy-rclpy.*]
+ignore_missing_imports = True

.python-version

@@ -0,0 +1 @@
+3.10.12

docker/.env

@@ -1,20 +1,24 @@
 COMPOSE_PROJECT_NAME=angel-system
 
+ROS_VERSION=iron
+
 # Location to root our installation and workspace in container space.
 # This must be an absolute path.
 ANGEL_WORKSPACE_DIR=/angel_workspace
 
 PTG_REGISTRY=gitlab.kitware.com:4567/ptg-angel/docker
-PTG_TAG=latest
+PTG_TAG=iron-upgrade
 
 # Directory for the workspace shell to mount files and directories into on the
 # host system. If a relative path, it will be interpreted as relative to the
 # docker-compose file.
 WORKSPACE_SHELL_HOST_DIR=../.workspace-shell-dev
 
-# Set this variable to 1 to cause ROS to only communicate to and receive from
-# localhost communications only.
-ROS_LOCALHOST_ONLY=1
+# Set this variable to "LOCALHOST" to cause ROS nodes to only discover other
+# nodes on the same machine.
+# Also see:
+#   https://docs.ros.org/en/iron/Releases/Release-Iron-Irwini.html#improved-discovery-options
+#ROS_AUTOMATIC_DISCOVERY_RANGE=LOCALHOST
 
 # If this variable is defined, then the standard workspace entrypoint will also
 # source the local workspace installation to ready the environment for using

docker/cyclonedds_profile.xml.tmpl

@@ -1,8 +1,13 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<CycloneDDS xmlns="https://cdds.io/config" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://cdds.io/config https://raw.githubusercontent.com/eclipse-cyclonedds/cyclonedds/master/etc/cyclonedds.xsd">
+<!-- See docs: https://github.com/eclipse-cyclonedds/cyclonedds/blob/0.10.3/docs/manual/options.md -->
+<CycloneDDS xmlns="https://cdds.io/config"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:schemaLocation="https://cdds.io/config https://raw.githubusercontent.com/eclipse-cyclonedds/cyclonedds/0.10.3/etc/cyclonedds.xsd">
     <Domain id="any">
         <General>
-            <NetworkInterfaceAddress>${CYCLONE_DDS_INTERFACE}</NetworkInterfaceAddress>
+            <Interfaces>
+                <NetworkInterface name="${CYCLONE_DDS_INTERFACE}"/>
+            </Interfaces>
         </General>
     </Domain>
 </CycloneDDS>

docker/docker-compose.yml

@@ -11,6 +11,8 @@ services:
     build:
       context: ..  # repo root
       dockerfile: docker/workspace-base-dev/Dockerfile
+      args:
+        ROS_VERSION:  # from .env file
       cache_from:
         # In order of descending precedence
         - ${PTG_REGISTRY}/workspace-base-dev:${PTG_TAG}
@@ -54,7 +56,7 @@ services:
       # Host sharing
       - ../poetry.lock:${ANGEL_WORKSPACE_DIR}/poetry.lock
       - ../pyproject.toml:${ANGEL_WORKSPACE_DIR}/pyproject.toml
-      - ../ros:${ANGEL_WORKSPACE_DIR}/src
+      - ../ros:${ANGEL_WORKSPACE_DIR}/ros
       - ../tmux:${ANGEL_WORKSPACE_DIR}/tmux
       - ../angel_system:${ANGEL_WORKSPACE_DIR}/angel_system
       - ../model_files:${ANGEL_WORKSPACE_DIR}/model_files
@@ -86,7 +88,7 @@ services:
     environment:
       HISTFILE: ${ANGEL_WORKSPACE_DIR}/stuff/bash_history
       SETUP_WORKSPACE_INSTALL:  # from .env file or parent environment
-      ROS_LOCALHOST_ONLY:  # from .env file or parent environment
+      ROS_AUTOMATIC_DISCOVERY_RANGE:  # from .env file or parent environment
       RMW_IMPLEMENTATION:  # from .env file or parent environment
       CYCLONE_DDS_INTERFACE:  # from .env file or parent environment
       DISPLAY:  # pull from current environment
@@ -122,7 +124,7 @@ services:
     volumes:
       - /dev/shm:/dev/shm
     environment:
-      ROS_LOCALHOST_ONLY:  # from .env file or parent environment
+      ROS_AUTOMATIC_DISCOVERY_RANGE:  # from .env file or parent environment
       RMW_IMPLEMENTATION:  # from .env file or parent environment
       CYCLONE_DDS_INTERFACE:  # from .env file or parent environment
       SETUP_WORKSPACE_INSTALL: 1  # yes, activate the workspace here

docker/workspace-base-dev/Dockerfile

@@ -4,7 +4,8 @@
 # This should include packages convenient for development, but not specifically
 # runtime requirements for the ROS2 system.
 #
-FROM osrf/ros:foxy-desktop AS base
+ARG ROS_VERSION
+FROM osrf/ros:${ROS_VERSION}-desktop AS base
 
 SHELL ["/bin/bash", "-c"]
 
@@ -16,6 +17,7 @@ RUN apt-get -y update \
         bash-completion \
         gdb \
         htop \
+        iproute2 \
         less \
         vim \
         parallel \

docker/workspace-build/Dockerfile

@@ -40,23 +40,25 @@ RUN apt-get -y update \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
-# For the demo-ui
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install -y nodejs && node -v && npm -v
-
 # Env vars for the nvidia-container-runtime.
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
 
+# For the demo-ui
+# TODO: Probably upgrade what nodejs version is being used.
+# TODO: Getting a warning about this script being deprecated, so update the installation method, too.
+RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
+RUN apt-get install -y nodejs && node -v && npm -v
+
 # Bring in poetry for standard python package installation.
 # Specifying a separate virtual environment that will be natively "activated".
 # NOTE: I have some misgivings about making "~/.local/" a "venv", but we have
 #       not observed adverse effects yet...
 ENV VIRTUAL_ENV=/root/.local
 ENV PATH=/root/.local/bin:${PATH}
-ENV POETRY_VERSION=1.5.1
+ENV POETRY_VERSION=1.7.1
 RUN python3 -m venv --system-site-packages "${VIRTUAL_ENV}"\
- && pip3 install pip==23.0 \
+ && pip3 install pip==23.3.1 \
  && (curl -sSL 'https://install.python-poetry.org' | python3 -) \
  && poetry config virtualenvs.create false
 
@@ -69,11 +71,11 @@ ENV ANGEL_WORKSPACE_DIR="${ANGEL_WORKSPACE_DIR}"
 # Isolate package.xml files for rosdep dependency installation.
 #
 FROM base AS tmp_package_files
-COPY ./ros "${ANGEL_WORKSPACE_DIR}"/src
+COPY ./ros "${ANGEL_WORKSPACE_DIR}"/ros
 RUN mkdir "${ANGEL_WORKSPACE_DIR}"/tmp \
- && (find "${ANGEL_WORKSPACE_DIR}"/src -type f -name 'package.xml' -print | xargs -n1 -I{} cp --parents {} "${ANGEL_WORKSPACE_DIR}"/tmp) \
- && rm -r "${ANGEL_WORKSPACE_DIR}"/src \
- && mv "${ANGEL_WORKSPACE_DIR}"/tmp/"${ANGEL_WORKSPACE_DIR}"/src "${ANGEL_WORKSPACE_DIR}"/ \
+ && (find "${ANGEL_WORKSPACE_DIR}"/ros -type f -name 'package.xml' -print | xargs -n1 -I{} cp --parents {} "${ANGEL_WORKSPACE_DIR}"/tmp) \
+ && rm -r "${ANGEL_WORKSPACE_DIR}"/ros \
+ && mv "${ANGEL_WORKSPACE_DIR}"/tmp/"${ANGEL_WORKSPACE_DIR}"/ros "${ANGEL_WORKSPACE_DIR}"/ \
  && rm -r "${ANGEL_WORKSPACE_DIR}"/tmp
 
 #
@@ -98,23 +100,33 @@ RUN cd "${ANGEL_WORKSPACE_DIR}" \
     # wheels and stuff.
  && rm -r ${HOME}/.cache/pip \
           ${HOME}/.cache/pypoetry
-RUN python3 -m nltk.downloader punkt
 
-# Build the angel_system python package
+# Create editable-installation hooks for the angel_system python package.
+# * The container is intended to be environment-only, but it is highly
+#   convenient to pre-establish the editable-installed package's metadata and
+#   .pth linkage within the container. This allows importing the *real*
+#   angel_system package from anywhere within the container post-mounting.
+#   It is burdensom to require users of the environment container to "install"
+#   the angel_system python package on every container instantiation.
 # * This is intentionally separated from the above dep installation so as to
 #   not trigger re-installation of all dependencies if there is only a change
 #   in the library and not with the libraries dependencies.
-COPY ./angel_system "${ANGEL_WORKSPACE_DIR}"/angel_system
 RUN cd "${ANGEL_WORKSPACE_DIR}" \
+ && mkdir "${ANGEL_WORKSPACE_DIR}"/angel_system \
+ && touch "${ANGEL_WORKSPACE_DIR}"/angel_system/__init__.py \
  && pip install --no-deps -e .
 
-# Build the ROS workspace.
-COPY ./ros "${ANGEL_WORKSPACE_DIR}/src"
-COPY docker/workspace_build_ros.sh "${ANGEL_WORKSPACE_DIR}"/
-RUN "${ANGEL_WORKSPACE_DIR}/workspace_build_ros.sh"
-
-# Add the "build NPM stuff" script.
-COPY docker/workspace_build_npm_install.sh "${ANGEL_WORKSPACE_DIR}"/
+## TODO: Remove -- environment only container
+## Build the ROS workspace.
+#COPY ./ros "${ANGEL_WORKSPACE_DIR}/src"
+#COPY docker/workspace_build_ros.sh "${ANGEL_WORKSPACE_DIR}"/
+#RUN "${ANGEL_WORKSPACE_DIR}/workspace_build_ros.sh"
+#
+## TODO: Remove -- environment only container
+## Build the required NPM envirnment for the Engineering UI.
+#COPY docker/workspace_build_npm_install.sh "${ANGEL_WORKSPACE_DIR}"/
+#RUN cd "${ANGEL_WORKSPACE_DIR}"/src/angel_utils/demo_ui/ \
+# && "${ANGEL_WORKSPACE_DIR}/workspace_build_npm_install.sh"
 
 # Bring in tmux run configurations
 # - add symlink to tmuxinator's expected "config home"

docker/workspace_build.sh

@@ -16,14 +16,6 @@
 set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-# We expect this to only be run manually, thus if we're not in a TTY session
-if ! tty -s
-then
-  echo "ERROR: Not running in a TTY. Are you sure you meant to run this?"
-  exit 1
-fi
-
-
 # We don't want to auto apt update every single time, so only do it once on
 # first invocation of this script. The /tmp directory is not expected to be
 # mounted to the host filesystem, so it should be "fresh" every instantiation

docker/workspace_build_npm_install.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
 # Workspace build component -- NPM component build / installation
-pushd "${ANGEL_WORKSPACE_DIR}"/src/angel_utils/multi_task_demo_ui
+pushd "${ANGEL_WORKSPACE_DIR}"/ros/angel_utils/multi_task_demo_ui
 npm install
 popd

docker/workspace_build_ros.sh

@@ -14,4 +14,4 @@ cd "$SCRIPT_DIR"
 # Activate the base ROS context and build our local workspace.
 # shellcheck disable=SC1090
 source "/opt/ros/${ROS_DISTRO}/setup.bash"
-colcon build --continue-on-error --merge-install "$@"
+colcon build --base-paths "${SCRIPT_DIR}/ros" --continue-on-error --merge-install "$@"

docker/workspace_build_rosdep_install.sh

@@ -7,4 +7,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # ROS2 workspace source directory.
 cd "$SCRIPT_DIR"
 
+# Run update if we have no cache yet
+if [[ ! -d ${HOME}/.ros/rosdep ]]
+then
+  rosdep update
+fi
+
 rosdep install -i --from-path "${ANGEL_WORKSPACE_DIR}" --rosdistro "${ROS_DISTRO}" -y

docker/workspace_setenv.sh

@@ -2,15 +2,9 @@
 # and run-time.
 
 # Enable multicast flag on the loopback device (should always be present)
-ifconfig lo multicast
+ip link set lo multicast on
 
-# Report on ROS localhost only mode.
-if [[ "${ROS_LOCALHOST_ONLY}" -ne 0 ]]
-then
-  >&2 echo "[INFO] Setting ROS to localhost-only communication."
-else
-  >&2 echo "[INFO] ROS host communication unconstrained."
-fi
+>&2 echo "[INFO] ROS_AUTOMATIC_DISCOVERY_RANGE: ${ROS_AUTOMATIC_DISCOVERY_RANGE}"
 
 # Configure RMW-specific configurations depending on what is set to be used.
 if [[ "${RMW_IMPLEMENTATION}" = "rmw_fastrtps_cpp" ]]

docs/index.rst

@@ -12,6 +12,7 @@ Welcome to ANGEL System's documentation!
 
    algorithms/index
    system/index
+   local_python_env
    python_tpl