From 75749da72412a822064daffdb13331c7d9f35a38 Mon Sep 17 00:00:00 2001 From: qhdwight Date: Mon, 18 Sep 2023 12:34:40 -0400 Subject: [PATCH] Fix style.sh, add venv to gitignore, update dockerfile, comment ansible, make dev.yml perform sysstem upgrade, make ansible handle python virtualenv instead of bootstrap, specify versions exactly in pyproject, format some code in auton --- .gitignore | 5 +- Dockerfile | 25 ++++----- ansible/roles/build/tasks/main.yml | 24 +++++++++ pyproject.toml | 26 +++++----- src/esw/__init__.py | 0 src/esw/brushless.py | 3 -- src/esw/cameras.py | 3 -- src/esw/imu_driver.py | 1 - src/esw/science.py | 1 - src/navigation/gate.py | 1 - src/navigation/partial_gate.py | 1 - src/navigation/state.py | 10 ++-- src/navigation/waypoint.py | 15 +++--- src/teleop/jetson/arm_trajectory_server.py | 4 -- style.sh | 59 +++++++++++++--------- 15 files changed, 96 insertions(+), 82 deletions(-) create mode 100644 src/esw/__init__.py diff --git a/.gitignore b/.gitignore index 9c7675372..2ac606dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,7 @@ logs/ moteus-cal* # CSV -*.csv \ No newline at end of file +*.csv + +# Virtual Environment +venv/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6d02b1f80..454b42b28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,26 @@ FROM ros:noetic -RUN apt-get update && apt-get upgrade -y -# Add apt repo for latest version of Git -RUN apt-get install software-properties-common -y && add-apt-repository ppa:git-core/ppa -y -RUN apt-get update && apt-get install -y \ - zsh neovim sudo git git-lfs \ - clang-format-12 clang-tidy-12 \ - python3-catkin-tools python3-pip -RUN DEBIAN_FRONTEND=noninteractive apt-get install keyboard-configuration -y +RUN apt-get update -y && apt-get upgrade -y && apt-get install software-properties-common -y +RUN apt-add-repository ppa:ansible/ansible -y +RUN apt-add-repository ppa:git-core/ppa -y +RUN apt install -y ansible git git-lfs RUN useradd --create-home --groups sudo --shell /bin/zsh mrover # Give mrover user sudo access with no password RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER mrover -RUN mkdir -p ~/catkin_ws/src/mrover +RUN mkdir -p /home/mrover/catkin_ws/src/mrover WORKDIR /home/mrover/catkin_ws/src/mrover -# ROS package manager (rosdep) reads this file to install dependencies ADD ./package.xml . -# Python package manager (pip) reads this file to install dependencies -ADD ./requirements.txt . -# Install ROS packages -RUN rosdep update && rosdep install --from-paths . --ignore-src -y --rosdistro=noetic +ADD ./pyproject.toml . +ADD ./ansible ./ansible +ADD ./ansible.sh . +RUN ./ansible.sh build.yml USER root # Remove apt cache to free up space in the image RUN apt-get clean && rm -rf /var/lib/apt/lists/* -# Install Python packags, sudo so it is a global install -RUN pip3 install -r ./requirements.txt USER mrover ENTRYPOINT [ "/bin/zsh" ] diff --git a/ansible/roles/build/tasks/main.yml b/ansible/roles/build/tasks/main.yml index 2c7444fd7..2565fc41e 100644 --- a/ansible/roles/build/tasks/main.yml +++ b/ansible/roles/build/tasks/main.yml @@ -1,8 +1,10 @@ +# Allows us to install Python versions newer than 3.8 - name: Add Python PPA become: True apt_repository: repo: ppa:deadsnakes/ppa +# Allows us to install G++11, so we can use an updated libstdc++ which provides more standard library features (C++20) - name: Add GCC PPA become: True apt_repository: @@ -13,6 +15,7 @@ apt_key: url: https://apt.llvm.org/llvm-snapshot.gpg.key +# Allows us to install Clang 16 and other LLVM tools - name: Add LLVM APT List become: True apt_repository: @@ -25,6 +28,7 @@ url: https://apt.kitware.com/keys/kitware-archive-latest.asc keyring: /usr/share/keyrings/kitware-archive-keyring.gpg +# Allows us to install CMake versions newer than 3.16 - name: Add CMake APT List become: True apt_repository: @@ -42,6 +46,13 @@ repo: deb http://packages.ros.org/ros/ubuntu {{ ubuntu_release }} main filename: ros +- name: Upgrade APT Packages + become: True + apt: + cache_valid_time: 604800 + state: latest + upgrade: yes + - name: Install APT Packages become: True apt: @@ -53,12 +64,16 @@ - neovim - sudo - cmake + # Caches intermediate build files, speeds up compilation over time - ccache + # Faster than make - ninja-build - tmux - zstd - htop - curl + - unzip + - rsync - python3-pip - python3-rosdep - python3-catkin-tools @@ -77,6 +92,7 @@ become: True command: rosdep init args: + # This command will be skipped if this file already exists creates: /etc/ros/rosdep/sources.list.d/20-default.list - name: Update rosdep @@ -133,3 +149,11 @@ path: /usr/bin/clang-16 link: /usr/bin/clang priority: 160 + +- name: Setup Python Virtual Environment + pip: + name: + # Installs from pyproject.toml + - "{{ catkin_workspace }}/src/mrover[dev]" + virtualenv: "{{ catkin_workspace }}/src/mrover/venv" + virtualenv_command: python3.10 -m venv diff --git a/pyproject.toml b/pyproject.toml index 138b596b7..abc2757c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,23 +9,23 @@ maintainers = [ { name = "Michigan Mars Rover Team" } ] dependencies = [ - "django", - "empy", - "numpy", - "pandas", - "shapely", - "pyserial", - "moteus", - "pymap3d", - "aenum", - "pyyaml", - "rospkg", + "Django==4.2.5", + "empy==3.3.4", + "numpy==1.26.0", + "pandas==2.1.0", + "shapely==2.0.1", + "pyserial==3.5", + "moteus==0.3.59", + "pymap3d==3.0.1", + "aenum==3.1.15", + "PyYAML==6.0.1", + "rospkg==1.5.0", ] [project.optional-dependencies] dev = [ - "black", - "mypy", + "black==23.9.1", + "mypy==1.5.1", ] [project.urls] diff --git a/src/esw/__init__.py b/src/esw/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/esw/brushless.py b/src/esw/brushless.py index 59b9b73ff..0cd9f40bc 100755 --- a/src/esw/brushless.py +++ b/src/esw/brushless.py @@ -125,12 +125,10 @@ def is_mode_indicating_error(mode: int) -> bool: class MoteusBridge: - MOTEUS_RESPONSE_TIME_INDICATING_DISCONNECTED_S = 0.01 ROVER_NODE_TO_MOTEUS_WATCHDOG_TIMEOUT_S = 0.1 def __init__(self, can_id: int, transport, gear_ratio: int): - self._can_id = can_id self.controller = moteus.Controller(id=can_id, transport=transport) self.command_lock = threading.Lock() @@ -369,7 +367,6 @@ async def send_command(self) -> None: for bridge in self._motor_bridges.values(): if lost_communication_now: - # If we only just lost communications this iteration. if not self._lost_communication: rospy.loginfo( diff --git a/src/esw/cameras.py b/src/esw/cameras.py index 3d7c20438..9ff860c5a 100755 --- a/src/esw/cameras.py +++ b/src/esw/cameras.py @@ -169,7 +169,6 @@ def __del__(self) -> None: class StreamManager: - MAX_STREAMS: int = rospy.get_param("cameras/max_streams") """ The maximum number of streams that can be simultaneously sustained. @@ -316,7 +315,6 @@ def handle_req(self, req: ChangeCamerasRequest) -> ChangeCamerasResponse: # If a stream is being requested... # (resolution == -1 means a request to cancel stream) if req.camera_cmd.resolution >= 0: - # If we cannot handle any more streams, return False. available_port_arr = [True, True, True, True] for i, stream in enumerate(self._stream_by_device): @@ -414,7 +412,6 @@ def send( # Transmit loop while True: - ret, frame = cap_send.read() if not ret: rospy.logerr("Empty frame") diff --git a/src/esw/imu_driver.py b/src/esw/imu_driver.py index 798f8c92a..22ea8b084 100755 --- a/src/esw/imu_driver.py +++ b/src/esw/imu_driver.py @@ -91,7 +91,6 @@ def main(): attempts = 0 while not rospy.is_shutdown(): - # try to read a line from the serial connection, # if this fails 100 times in a row then end the program try: diff --git a/src/esw/science.py b/src/esw/science.py index 71ed6cdbe..a77cdf69b 100755 --- a/src/esw/science.py +++ b/src/esw/science.py @@ -90,7 +90,6 @@ class ScienceBridge: _last_mcu_active: Bool def __init__(self) -> None: - self._active_publisher = rospy.Publisher("science_mcu_active", Bool, queue_size=1) self._mcu_active_timeout_s = rospy.get_param("science/info/mcu_active_timeout_s") diff --git a/src/navigation/gate.py b/src/navigation/gate.py index 3635b7faa..3638e8004 100644 --- a/src/navigation/gate.py +++ b/src/navigation/gate.py @@ -216,7 +216,6 @@ class GateTraverseStateTransitions(Enum): class GateTraverseState(BaseState): - STOP_THRESH = get_rosparam("gate/stop_thresh", 0.2) DRIVE_FWD_THRESH = get_rosparam("gate/drive_fwd_thresh", 0.34) # 20 degrees APPROACH_DISTANCE = get_rosparam("gate/approach_distance", 2.0) diff --git a/src/navigation/partial_gate.py b/src/navigation/partial_gate.py index e76f2e44d..88e1df022 100644 --- a/src/navigation/partial_gate.py +++ b/src/navigation/partial_gate.py @@ -64,7 +64,6 @@ class PartialGateStateTransitions(Enum): class PartialGateState(BaseState): - traj: Optional[PartialGateTrajectory] = None def __init__( diff --git a/src/navigation/state.py b/src/navigation/state.py index 23c1e8926..c35295f33 100644 --- a/src/navigation/state.py +++ b/src/navigation/state.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import List +from typing import List, Optional import smach from context import Context @@ -19,9 +19,9 @@ def __init__( self, context: Context, own_transitions: List[str], - add_outcomes: List[str] = None, - add_input_keys: List[str] = None, - add_output_keys: List[str] = None, + add_outcomes: Optional[List[str]] = None, + add_input_keys: Optional[List[str]] = None, + add_output_keys: Optional[List[str]] = None, ): add_outcomes = add_outcomes or [] add_input_keys = add_input_keys or [] @@ -72,7 +72,7 @@ def reset(self): def evaluate(self, ud: smach.UserData) -> str: """Override me instead of execute!""" - pass + raise NotImplementedError class DoneStateTransitions(Enum): diff --git a/src/navigation/waypoint.py b/src/navigation/waypoint.py index d86cc8029..98e6a5aac 100644 --- a/src/navigation/waypoint.py +++ b/src/navigation/waypoint.py @@ -1,11 +1,10 @@ -from typing import List - -import numpy as np -import rospy +from typing import List, Optional import tf2_ros -from context import Context, Environment + +import numpy as np from aenum import Enum, NoAlias +from context import Context from state import BaseState from util.ros_utils import get_rosparam @@ -31,9 +30,9 @@ class WaypointState(BaseState): def __init__( self, context: Context, - add_outcomes: List[str] = None, - add_input_keys: List[str] = None, - add_output_keys: List[str] = None, + add_outcomes: Optional[List[str]] = None, + add_input_keys: Optional[List[str]] = None, + add_output_keys: Optional[List[str]] = None, ): add_outcomes = add_outcomes or [] add_input_keys = add_input_keys or [] diff --git a/src/teleop/jetson/arm_trajectory_server.py b/src/teleop/jetson/arm_trajectory_server.py index d91dcdce2..94a213ad8 100755 --- a/src/teleop/jetson/arm_trajectory_server.py +++ b/src/teleop/jetson/arm_trajectory_server.py @@ -74,17 +74,14 @@ def error_threshold_exceeded(feedback: FollowJointTrajectoryFeedback) -> str: class MoveItAction(object): - # Rearranges the point path following the name convention joint_0, ... joint_6 def rearrange(self, joint_trajectory: JointTrajectory) -> None: - mapping = [joint_trajectory.joint_names.index(j) for j in conf_joint_names] # Return early if already arranged properly if mapping == sorted(mapping): return for point in joint_trajectory.points: - temp_positions: List[float] = [] temp_velocities: List[float] = [] temp_accelerations: List[float] = [] @@ -121,7 +118,6 @@ def __init__(self, name: str) -> None: # Action callback def execute_cb(self, goal: FollowJointTrajectoryGoal) -> None: - rospy.loginfo("Executing FollowJointTrajectory Action") # It is required to rearrange the arrays because MoveIt doesn't guarantee order preservation self.rearrange(goal.trajectory) diff --git a/style.sh b/style.sh index 0ca03c21b..2afe1daf3 100755 --- a/style.sh +++ b/style.sh @@ -3,58 +3,67 @@ # See: https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ set -Eeuo pipefail -RED='\033[0;31m' -NC='\033[0m' +readonly RED='\033[0;31m' +readonly NC='\033[0m' + +print_update_error() { + echo -e "${RED}[Error] Please update with ./ansible.sh build.yml${NC}" + exit 1 +} ## Check that all tools are installed -clang_format_executable=clang-format-16 -clang_format_executable_path=$(which "$clang_format_executable") +readonly clang_format_executable=clang-format-16 +readonly clang_format_executable_path=$(which "$clang_format_executable") if [ ! -x "$clang_format_executable_path" ]; then - echo -e "${RED}[Error] Please install clang-format with: sudo apt install ${clang_format_executable}${NC}" - exit 1 + echo -e "${RED}[Error] Could not find clang-format-16${NC}" + print_update_error fi -black_executable=black -black_executable_path=$(which "$black_executable") +readonly black_executable=black +readonly black_executable_path=$(which "$black_executable") if [ ! -x "$black_executable_path" ]; then - echo -e "${RED}[Error] Please run pip3 install -r requirements.txt${NC}" - exit 1 + echo -e "${RED}[Error] Could not find black${NC}" + print_update_error fi # Style check Python with black -if ! black --version | grep -q 'black, 22.8.0'; then +if ! black --version | grep -q 'black, 23.9.1'; then echo -e "${RED}[Error] Wrong black version${NC}" - exit 1 + print_update_error fi -mypy_executable=mypy -mypy_executable_path=$(which "$mypy_executable") +readonly mypy_executable=mypy +readonly mypy_executable_path=$(which "$mypy_executable") if [ ! -x "$mypy_executable_path" ]; then - echo -e "${RED}[Error] Please run pip3 install -r requirements.txt${NC}" - exit 1 + echo -e "${RED}[Error] Could not find mypy${NC}" + print_update_error fi -if ! mypy --version | grep -q 'mypy 0.971'; then +if ! mypy --version | grep -q 'mypy 1.5.1'; then echo -e "${RED}[Error] Wrong mypy version${NC}" - exit 1 + print_update_error fi ## Run checks -# Fail immediately if any command below fails -set -Eeo pipefail - echo "Style checking C++ ..." -find ./src -regex '.*\.\(cpp\|hpp\|h\)' -exec "$clang_format_executable_path" --dry-run -style=file -i {} \; +readonly FOLDERS=( + ./src/perception + ./src/gazebo + ./src/util +) +for folder in "${FOLDERS[@]}"; do + find "$folder" -regex '.*\.\(cpp\|hpp\|h\)' -exec "$clang_format_executable_path" --dry-run -style=file -i {} \; +done echo "Done" +echo echo "Style checking Python with black ..." -"$black_executable_path" --check --diff --line-length=120 ./src ./scripts -echo "Done" +"$black_executable_path" --line-length=120 ./src ./scripts +echo echo "Style checking Python with mypy ..." "$mypy_executable_path" --config-file mypy.ini --check ./src ./scripts -echo "Done"