ViNT v1 Release: Checkpoints, Training, LoCoBot Deployment

.gitignore

@@ -1,3 +1,22 @@
+train/logs/*
+train/datasets/*
+train/vint_train/data/data_splits/*
+train/wandb/*
+train/gnm_dataset/*
+
+*.png
+*.jpg
+*.pth
+*.mp4
+*.gif
+
+deployment/model_weights/*
+deployment/topomaps/*
+
+.vscode/*
+*/.vscode/*
+
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -20,6 +39,7 @@ parts/
 sdist/
 var/
 wheels/
+pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -49,7 +69,6 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
-cover/
 
 # Translations
 *.mo
@@ -72,7 +91,6 @@ instance/
 docs/_build/
 
 # PyBuilder
-.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -83,9 +101,7 @@ profile_default/
 ipython_config.py
 
 # pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -94,22 +110,7 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
 # Celery stuff
@@ -145,16 +146,3 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Dhruv Shah
+Copyright (c) 2022 Dhruv Shah, Ajay Sridhar, Noriaki Hirose, Sergey Levine
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

deployment/config/camera_front.yaml

@@ -0,0 +1,8 @@
+# camera parameters for src/gnm_locobot.launch
+video_device: "/dev/video0" # change this to your video device path
+image_width: 160 #640
+image_height: 120 #480
+pixel_format: yuyv
+camera_frame_id: "usb_cam"
+io_method: "mmap"
+framerate: 9

deployment/config/camera_reverse.yaml

@@ -0,0 +1,8 @@
+# camera parameters for src/gnm_locobot.launch
+video_device: "/dev/video2" # change this to your video device path
+image_width: 160
+image_height: 120
+pixel_format: yuyv
+camera_frame_id: "usb_cam"
+io_method: "mmap"
+framerate: 9

deployment/config/cmd_vel_mux.yaml

@@ -0,0 +1,38 @@
+# # Velocity command sources
+# vel_sources:
+#   # Teleoperation commands
+#   teleop:
+#     topic: /cmd_vel_mux/input/teleop
+#     timeout: 0.6
+#     priority: 100
+
+#   # Move base commands
+#   move_base:
+#     topic: /cmd_vel_mux/input/navi
+#     timeout: 0.6
+#     priority: 90
+
+# # Mux parameters
+# yaml_cfg_file: ""
+# allow_unsafe_topics: false
+# cmd_vel_timeout: 0.25
+# publish_topic: /mobile_base/commands/velocity
+
+
+subscribers:
+  - name:        "gnm vels"
+    topic:       "/cmd_vel_mux/input/navi"
+    timeout:     0.1
+    priority:    0
+    short_desc:  "The default cmd_vel, controllers unaware that we are multiplexing cmd_vel should come here"
+  - name:        "teleop"
+    topic:       "/cmd_vel_mux/input/teleop"
+    timeout:     0.5
+    priority:    2
+    short_desc:  "Navigation stack controller"
+  - name:       "gnm recovery"
+    topic:       "/cmd_vel_mux/input/recovery"
+    timeout:     0.1
+    priority:    1
+publisher:       "/mobile_base/commands/velocity"
+

deployment/config/joystick.yaml

@@ -0,0 +1,7 @@
+# joystick parameters for src/gnm_locobot.launch
+dev: "/dev/input/js0" # change this to your joystick device path
+
+# joystick parameters for src/joy_teleop.py
+deadman_switch: 5 # button index
+lin_vel_button: 4
+ang_vel_button: 0 

deployment/config/models.yaml

@@ -0,0 +1,18 @@
+vint: 
+  config_path: "../../train/config/vint.yaml"
+  ckpt_path: "../model_weights/vint5c_29.pth"
+
+# gnmv9: 
+#   config_path: "../../train/config/gnmv9.yaml"
+#   ckpt_path: "../model_weights/gnmv9_29.pth"
+
+late_fusion: 
+  config_path: "../../train/config/late_fusion.yaml"
+  ckpt_path: "../model_weights/vint_late_fusion.pth"
+
+gnm: 
+  config_path: "../../train/config/gnm.yaml"
+  ckpt_path: "../model_weights/gnm_large.pth"
+
+
+# add your own model configs here after saving the *.pth file to ../model_weight

deployment/config/robot.yaml

@@ -0,0 +1,13 @@
+# linear and angular speed limits for the robot
+max_v: 0.2 #0.4 # m/s
+max_w: 0.4 #0.8 # rad/s
+# observation rate fo the robot
+frame_rate: 4 # Hz
+graph_rate: 0.3333 # Hz  
+
+# topic names (modify for different robots/nodes)
+vel_teleop_topic: /cmd_vel_mux/input/teleop 
+vel_navi_topic: /cmd_vel_mux/input/navi
+vel_recovery_topic: /cmd_vel_mux/input/recovery
+
+

deployment/deployment_environment.yml

@@ -0,0 +1,17 @@
+name: vint_deployment
+channels:
+- pytorch
+- conda-forge
+dependencies:
+- python=3.8.5
+- cudatoolkit=11.
+- torchvision
+- numpy
+- matplotlib
+- pyyaml
+- rospkg
+- pip:
+  - torch
+  - torchvision
+  - efficientnet_pytorch
+  - warmup_scheduler

deployment/src/create_topomap.py

@@ -0,0 +1,93 @@
+import argparse
+import os
+from utils import msg_to_pil 
+import time
+
+# ROS
+import rospy
+from sensor_msgs.msg import Image
+from sensor_msgs.msg import Joy
+
+IMAGE_TOPIC = "/usb_cam/image_raw"
+TOPOMAP_IMAGES_DIR = "../topomaps/images"
+obs_img = None
+
+
+def remove_files_in_dir(dir_path: str):
+    for f in os.listdir(dir_path):
+        file_path = os.path.join(dir_path, f)
+        try:
+            if os.path.isfile(file_path) or os.path.islink(file_path):
+                os.unlink(file_path)
+            elif os.path.isdir(file_path):
+                shutil.rmtree(file_path)
+        except Exception as e:
+            print("Failed to delete %s. Reason: %s" % (file_path, e))
+
+
+def callback_obs(msg: Image):
+    global obs_img
+    obs_img = msg_to_pil(msg)
+
+
+def callback_joy(msg: Joy):
+    if msg.buttons[0]:
+        rospy.signal_shutdown("shutdown")
+
+
+def main(args: argparse.Namespace):
+    global obs_img
+    rospy.init_node("CREATE_TOPOMAP", anonymous=False)
+    image_curr_msg = rospy.Subscriber(
+        IMAGE_TOPIC, Image, callback_obs, queue_size=1)
+    subgoals_pub = rospy.Publisher(
+        "/subgoals", Image, queue_size=1)
+    joy_sub = rospy.Subscriber("joy", Joy, callback_joy)
+
+    topomap_name_dir = os.path.join(TOPOMAP_IMAGES_DIR, args.dir)
+    if not os.path.isdir(topomap_name_dir):
+        os.makedirs(topomap_name_dir)
+    else:
+        print(f"{topomap_name_dir} already exists. Removing previous images...")
+        remove_files_in_dir(topomap_name_dir)
+
+
+    assert args.dt > 0, "dt must be positive"
+    rate = rospy.Rate(1/args.dt)
+    print("Registered with master node. Waiting for images...")
+    i = 0
+    start_time = float("inf")
+    while not rospy.is_shutdown():
+        if obs_img is not None:
+            obs_img.save(os.path.join(topomap_name_dir, f"{i}.png"))
+            print("published image", i)
+            i += 1
+            rate.sleep()
+            start_time = time.time()
+            obs_img = None
+        if time.time() - start_time > 2 * args.dt:
+            print(f"Topic {IMAGE_TOPIC} not publishing anymore. Shutting down...")
+            rospy.signal_shutdown("shutdown")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=f"Code to generate topomaps from the {IMAGE_TOPIC} topic"
+    )
+    parser.add_argument(
+        "--dir",
+        "-d",
+        default="topomap",
+        type=str,
+        help="path to topological map images in ../topomaps/images directory (default: topomap)",
+    )
+    parser.add_argument(
+        "--dt",
+        "-t",
+        default=1.,
+        type=float,
+        help=f"time between images sampled from the {IMAGE_TOPIC} topic (default: 3.0)",
+    )
+    args = parser.parse_args()
+
+    main(args)

deployment/src/create_topomap.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Create a new tmux session
+session_name="gnm_locobot_$(date +%s)"
+tmux new-session -d -s $session_name
+
+# Split the window into three panes
+tmux selectp -t 0    # select the first (0) pane
+tmux splitw -v -p 50 # split it into two halves
+tmux selectp -t 0    # go back to the first pane
+tmux splitw -h -p 50 # split it into two halves
+
+# Run roscore in the first pane
+tmux select-pane -t 0
+tmux send-keys "roscore" Enter
+
+# Run the create_topoplan.py script with command line args in the second pane
+tmux select-pane -t 1
+tmux send-keys "conda activate gnm_deployment" Enter
+tmux send-keys "python create_topomap.py --dt 1 --dir $1" Enter
+
+# Change the directory to ../topomaps/bags and run the rosbag play command in the third pane
+tmux select-pane -t 2
+tmux send-keys "mkdir -p ../topomaps/bags" Enter
+tmux send-keys "cd ../topomaps/bags" Enter
+tmux send-keys "rosbag play -r 1.5 $2" # feel free to change the playback rate to change the edge length in the graph
+
+# Attach to the tmux session
+tmux -2 attach-session -t $session_name