Imitation learning with dagger

flow/controllers/imitation_learning/bottleneck_env.py

@@ -0,0 +1,150 @@
+"""Bottleneck example.
+Bottleneck in which the actions are specifying a desired velocity
+in a segment of space
+"""
+from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \
+    InFlows, SumoCarFollowingParams, SumoLaneChangeParams
+from flow.core.params import TrafficLightParams
+from flow.core.params import VehicleParams
+from flow.controllers import RLController, ContinuousRouter, \
+    SimLaneChangeController
+from flow.envs import BottleneckDesiredVelocityEnv
+from flow.networks import BottleneckNetwork
+
+# time horizon of a single rollout
+HORIZON = 1000
+# number of parallel workers
+N_CPUS = 2
+# number of rollouts per training iteration
+N_ROLLOUTS = N_CPUS * 4
+
+SCALING = 1
+NUM_LANES = 4 * SCALING  # number of lanes in the widest highway
+DISABLE_TB = True
+DISABLE_RAMP_METER = True
+AV_FRAC = 0.10
+
+vehicles = VehicleParams()
+vehicles.add(
+    veh_id="human",
+    lane_change_controller=(SimLaneChangeController, {}),
+    routing_controller=(ContinuousRouter, {}),
+    car_following_params=SumoCarFollowingParams(
+        speed_mode="all_checks",
+    ),
+    lane_change_params=SumoLaneChangeParams(
+        lane_change_mode=0,
+    ),
+    num_vehicles=1 * SCALING)
+vehicles.add(
+    veh_id="followerstopper",
+    acceleration_controller=(RLController, {}),
+    lane_change_controller=(SimLaneChangeController, {}),
+    routing_controller=(ContinuousRouter, {}),
+    car_following_params=SumoCarFollowingParams(
+        speed_mode=9,
+    ),
+    lane_change_params=SumoLaneChangeParams(
+        lane_change_mode=0,
+    ),
+    num_vehicles=1 * SCALING)
+
+controlled_segments = [("1", 1, False), ("2", 2, True), ("3", 2, True),
+                       ("4", 2, True), ("5", 1, False)]
+num_observed_segments = [("1", 1), ("2", 3), ("3", 3), ("4", 3), ("5", 1)]
+additional_env_params = {
+    "target_velocity": 40,
+    "disable_tb": True,
+    "disable_ramp_metering": True,
+    "controlled_segments": controlled_segments,
+    "symmetric": False,
+    "observed_segments": num_observed_segments,
+    "reset_inflow": False,
+    "lane_change_duration": 5,
+    "max_accel": 3,
+    "max_decel": 3,
+    "inflow_range": [1000, 2000]
+}
+
+# flow rate
+flow_rate = 2300 * SCALING
+
+# percentage of flow coming out of each lane
+inflow = InFlows()
+inflow.add(
+    veh_type="human",
+    edge="1",
+    vehs_per_hour=flow_rate * (1 - AV_FRAC),
+    departLane="random",
+    departSpeed=10)
+inflow.add(
+    veh_type="followerstopper",
+    edge="1",
+    vehs_per_hour=flow_rate * AV_FRAC,
+    departLane="random",
+    departSpeed=10)
+
+traffic_lights = TrafficLightParams()
+if not DISABLE_TB:
+    traffic_lights.add(node_id="2")
+if not DISABLE_RAMP_METER:
+    traffic_lights.add(node_id="3")
+
+additional_net_params = {"scaling": SCALING, "speed_limit": 23}
+net_params = NetParams(
+    inflows=inflow,
+    additional_params=additional_net_params)
+
+flow_params = dict(
+    # name of the experiment
+    exp_tag="DesiredVelocity",
+
+    # name of the flow environment the experiment is running on
+    env_name=BottleneckDesiredVelocityEnv,
+
+    # name of the network class the experiment is running on
+    network=BottleneckNetwork,
+
+    # simulator that is used by the experiment
+    simulator='traci',
+
+    # sumo-related parameters (see flow.core.params.SumoParams)
+    sim=SumoParams(
+        sim_step=0.5,
+        render=False,
+        print_warnings=False,
+        restart_instance=True,
+    ),
+
+    # environment related parameters (see flow.core.params.EnvParams)
+    env=EnvParams(
+        warmup_steps=40,
+        sims_per_step=1,
+        horizon=HORIZON,
+        additional_params=additional_env_params,
+    ),
+
+    # network-related parameters (see flow.core.params.NetParams and the
+    # network's documentation or ADDITIONAL_NET_PARAMS component)
+    net=NetParams(
+        inflows=inflow,
+        additional_params=additional_net_params,
+    ),
+
+    # vehicles to be placed in the network at the start of a rollout (see
+    # flow.core.params.VehicleParams)
+    veh=vehicles,
+
+    # parameters specifying the positioning of vehicles upon initialization/
+    # reset (see flow.core.params.InitialConfig)
+    initial=InitialConfig(
+        spacing="uniform",
+        min_gap=5,
+        lanes_distribution=float("inf"),
+        edges_distribution=["2", "3", "4", "5"],
+    ),
+
+    # traffic lights to be introduced to specific nodes (see
+    # flow.core.params.TrafficLightParams)
+    tls=traffic_lights,
+)

flow/controllers/imitation_learning/imitating_controller.py

@@ -0,0 +1,104 @@
+import numpy as np
+import tensorflow as tf
+from utils import *
+import tensorflow_probability as tfp
+from flow.controllers.base_controller import BaseController
+from replay_buffer import ReplayBuffer
+
+
+class ImitatingController(BaseController):
+    """
+    Controller which learns to imitate another given expert controller.
+    """
+    # Implementation in Tensorflow
+
+    def __init__(self, veh_id, sess, action_dim, obs_dim, num_layers, size, learning_rate, replay_buffer_size, training = True, inject_noise=0, noise_variance=0.5, policy_scope='policy_vars', car_following_params=None, time_delay=0.0, noise=0, fail_safe=None):
+
+        BaseController.__init__(self, veh_id, car_following_params, delay=time_delay, fail_safe=fail_safe, noise=noise)
+        self.sess = sess
+        self.action_dim = action_dim
+        self.obs_dim = obs_dim
+        self.num_layers = num_layers
+        self.size = size
+        self.learning_rate = learning_rate
+        self.training = training
+        self.inject_noise=inject_noise
+        self.noise_variance = noise_variance
+
+        with tf.variable_scope(policy_scope, reuse=tf.AUTO_REUSE):
+            self.build_network()
+
+
+        if self.training:
+            self.replay_buffer = ReplayBuffer(replay_buffer_size)
+        else:
+            self.replay_buffer = None
+
+        self.policy_vars = [v for v in tf.all_variables() if policy_scope in v.name and 'train' not in v.name]
+        self.saver = tf.train.Saver(self.policy_vars, max_to_keep=None)
+
+    def build_network(self):
+        """
+        Defines neural network for choosing actions.
+        """
+        self.define_placeholders()
+        self.define_forward_pass()
+        if self.training:
+            with tf.variable_scope('train', reuse=tf.AUTO_REUSE):
+                self.define_train_op()
+
+
+    def define_placeholders(self):
+        """
+        Defines input, output, and training placeholders for neural net
+        """
+        self.obs_placeholder = tf.placeholder(shape=[None, self.obs_dim], name="obs", dtype=tf.float32)
+        self.action_placeholder = tf.placeholder(shape=[None, self.action_dim], name="action", dtype=tf.float32)
+
+        if self.training:
+            self.action_labels_placeholder = tf.placeholder(shape=[None, self.action_dim], name="labels", dtype=tf.float32)
+
+    def define_forward_pass(self):
+        pred_action = build_neural_net(self.obs_placeholder, output_size=self.action_dim, scope='network_scope', n_layers=self.num_layers, size=self.size)
+        self.action_predictions = pred_action
+        print("TYPE: ", type(self.obs_placeholder))
+
+        if self.inject_noise == 1:
+            self.action_predictions = self.action_predictions + tf.random_normal(tf.shape(self.action_predictions), 0, self.noise_variance)
+
+    def define_train_op(self):
+        true_actions = self.action_labels_placeholder
+        predicted_actions = self.action_predictions
+
+        self.loss = tf.losses.mean_squared_error(true_actions, predicted_actions)
+        self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)
+
+    def train(self, observation_batch, action_batch):
+        action_batch = action_batch.reshape(action_batch.shape[0], self.action_dim)
+        ret = self.sess.run([self.train_op, self.loss], feed_dict={self.obs_placeholder: observation_batch, self.action_labels_placeholder: action_batch})
+
+    def get_accel_from_observation(self, observation):
+        # network expects an array of arrays (matrix); if single observation (no batch), convert to array of arrays
+        if len(observation.shape)<=1:
+            observation = observation[None]
+        ret_val = self.sess.run([self.action_predictions], feed_dict={self.obs_placeholder: observation})[0]
+
+        return ret_val
+
+    def get_accel(self, env):
+        # network expects an array of arrays (matrix); if single observation (no batch), convert to array of arrays
+        observation = env.get_state()
+        return self.get_accel_from_observation(observation)
+
+    def add_to_replay_buffer(self, rollout_list):
+        """ Add rollouts to replay buffer """
+
+        self.replay_buffer.add_rollouts(rollout_list)
+
+    def sample_data(self, batch_size):
+        """ Sample a batch of data from replay buffer """
+
+        return self.replay_buffer.sample_batch(batch_size)
+
+    def save_network(self, save_path):
+        self.saver.save(self.sess, save_path)

flow/controllers/imitation_learning/replay_buffer.py

@@ -0,0 +1,64 @@
+import time
+import numpy as np
+import tensorflow as tf
+import gym
+import os
+from utils import *
+
+
+class ReplayBuffer(object):
+    """ Replay buffer class to store state, action, expert_action, reward, next_state, terminal tuples"""
+
+    def __init__(self, max_size=100000):
+
+        self.max_size = max_size
+
+        # store each rollout
+        self.rollouts = []
+
+        # store component arrays from each rollout
+        self.observations = None
+        self.actions = None
+        self.expert_actions = None
+        self.rewards = None
+        self.next_observations = None
+        self.terminals = None
+
+
+    def add_rollouts(self, rollouts_list):
+        """
+        Add a list of rollouts to the replay buffer
+        """
+
+        for rollout in rollouts_list:
+            self.rollouts.append(rollout)
+
+        observations, actions, expert_actions, rewards, next_observations, terminals = unpack_rollouts(rollouts_list)
+
+        assert (not np.any(np.isnan(expert_actions))), "Invalid actions added to replay buffer"
+
+        if self.observations is None:
+            self.observations = observations[-self.max_size:]
+            self.actions = actions[-self.max_size:]
+            self.expert_actions = expert_actions[-self.max_size:]
+            self.rewards = rewards[-self.max_size:]
+            self.next_observations = next_observations[-self.max_size:]
+            self.terminals = terminals[-self.max_size:]
+        else:
+            self.observations = np.concatenate([self.observations, observations])[-self.max_size:]
+            self.actions = np.concatenate([self.actions, actions])[-self.max_size:]
+            self.expert_actions = np.concatenate([self.expert_actions, expert_actions])[-self.max_size:]
+            self.rewards = np.concatenate([self.rewards, rewards])[-self.max_size:]
+            self.next_observations = np.concatenate([self.next_observations, next_observations])[-self.max_size:]
+            self.terminals = np.concatenate([self.terminals, terminals])[-self.max_size:]
+
+    def sample_batch(self, batch_size):
+        """
+            Sample a batch of data (with size batch_size) from replay buffer.
+            Returns data in separate numpy arrays of observations, actions, rewards, next_observations, terminals
+        """
+        assert self.observations is not None and self.actions is not None and self.expert_actions is not None and self.rewards is not None and self.next_observations is not None and self.terminals is not None
+
+        size = len(self.observations)
+        rand_inds = np.random.randint(0, size, batch_size)
+        return self.observations[rand_inds], self.actions[rand_inds], self.expert_actions[rand_inds], self.rewards[rand_inds], self.next_observations[rand_inds], self.terminals[rand_inds]