Merge pull request #138 from BlueAndi/feature/RL

Implementation of data exchange mechanisms between robot and supervisor using Serial Webots drivers

.github/workflows/main.yml

@@ -148,7 +148,7 @@ jobs:
     needs: intro
     strategy:
       matrix:
-        environment: ["CalibTarget", "ConvoyFollowerTarget", "ConvoyLeaderTarget", "LineFollowerTarget", "LineFollowerSimpleTarget", "RemoteControlTarget", "SensorFusionTarget", "CalibSim", "ConvoyFollowerSim", "ConvoyLeaderSim", "LineFollowerSim", "LineFollowerSimpleSim", "RemoteControlSim", "SensorFusionSim"]
+        environment: ["CalibTarget", "ConvoyFollowerTarget", "ConvoyLeaderTarget", "LineFollowerTarget", "LineFollowerSimpleTarget", "RemoteControlTarget", "SensorFusionTarget", "CalibSim", "ConvoyFollowerSim", "ConvoyLeaderSim", "LineFollowerSim", "LineFollowerSimpleSim", "ReinforcementLearningSim", "RemoteControlSim", "SensorFusionSim"]
 
     steps:
     - name: Checkout repository
@@ -212,7 +212,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install pylint
+          pip install pylint "git+https://github.com/gabryelreyes/SerialMuxProt.git#egg=SerialMuxProt&subdirectory=python/SerialMuxProt"
 
       - name: Analysing the code with pylint
         run: |

README.md

@@ -13,6 +13,7 @@ Several kind of exclusive applications are available:
 * Line Follower - Just a line follower, using a PID controller.
 * Remote Control - The robot is remote controlled by e.g. the [DroidControlShip](https://github.com/BlueAndi/DroidControlShip) in a convoy follower role.
 * Sensor Fusion - The robot provides odometry and inertial data to the [DroidControlShip](https://github.com/BlueAndi/DroidControlShip), which calculates the sensor fusion based location information.
+* Line Follower with Reinforcement Learning - A line follower with reinforcement learning uses a learning agent that uses rewards and punishments to steer optimally in order to follow a line autonomously.
 
 ## Table of content
 
@@ -158,6 +159,7 @@ Example for the **LineFollowerTarget** application:
 | LineFollowerSimple | Just a simple line follower, using a PID controller. | Yes | No | ./webots/worlds/ETrack.wbt  ./webots/worlds/LargeTrack.wbt  ./webots/worlds/LineFollowerTrack.wbt |
 | RemoteControl | The robot is remote controlled by e.g. the [DroidControlShip](https://github.com/BlueAndi/DroidControlShip) in a convoy follower role. | No | Yes | ./webots/world/zumo_with_com_system/* |
 | SensorFusion | The robot provides odometry and inertial data to the [DroidControlShip](https://github.com/BlueAndi/DroidControlShip), which calculates the sensor fusion based location information. | No | Yes | ./webots/worlds/zumo_with_com_system/LineFollowerTrack.wbt |
+| ReinforcementLearning | A line follower with reinforcement learning uses a learning agent that uses rewards and punishments to steer optimally in order to follow a line autonomously. | Yes | No | ./webots/worlds/RL_LineFollower.wbt |
 | Test | Only for testing purposes on native environment. | Yes | No | N/A |
 
 # Documentation

doc/architecture/uml/LogicalView/ReinforcementLearning/Application.plantuml

@@ -0,0 +1,96 @@
+@startuml
+
+title Application
+
+package "Application" as appLayer {
+
+    class App <<main>> {
+        + setup() : void
+        + loop() : void
+    }
+
+    note left of App
+        The program entry point.
+    end note
+
+    class StateMachine <<control>> {
+        + setState(state : IState*) : void
+        + getState() : IState*
+        + process() : void
+    }
+
+    note left of StateMachine
+        The state machine executes always one
+        state exclusive. It is responsible to
+        enter/leave/process a state.
+    end note
+
+    interface IState {
+        + {abstract} entry() : void
+        + {abstract} process(sm : StateMachine&) : void
+        + {abstract} exit() : void
+    }
+
+    note left of IState
+        Defines the abstract state interface,
+        which forces every inherited class
+        to realize it.
+    end note
+
+    class StartupState <<control>>
+    class MotorSpeedCalibrationState <<control>>
+    class LineSensorsCalibrationState <<control>>
+    class ErrorState <<control>>
+    class DrivingState <<control>>
+    class ReadyState <<control>>
+
+    note bottom of StartupState
+        The system starts up and shows
+        the Application name on the display.
+    end note
+
+    note bottom of MotorSpeedCalibrationState
+        The robot drives with full speed forward
+        and with full speed backwards to determine
+        the max speed in steps/s. The slowest
+        motor is considered!
+    end note
+
+    note bottom of LineSensorsCalibrationState
+        The robot turns several times the
+        line sensors over the track for
+        calibration.
+    end note
+
+    note bottom of ErrorState
+        Error information is shown on display.
+        Confirmation from operator is requested.
+    end note
+
+    note bottom of DrivingState
+        The robot follows the line.
+    end note
+
+    note bottom of ReadyState
+        The robot is stopped and waits for
+        operator input.
+    end note
+}
+
+note top of  appLayer
+    Hint: See the application state behaviour
+    in the corresponding state diagram.
+end note
+
+App *--> StateMachine
+StateMachine o--> "0..1" IState
+
+IState <|.. StartupState: <<realize>>
+IState <|.... MotorSpeedCalibrationState: <<realize>>
+IState <|.. LineSensorsCalibrationState: <<realize>>
+IState <|.... ErrorState: <<realize>>
+IState <|.. ReadyState: <<realize>>
+IState <|.... DrivingState: <<realize>>
+
+
+@enduml

doc/architecture/uml/LogicalView/ReinforcementLearning/DrivingState.plantuml

@@ -0,0 +1,104 @@
+@startuml
+
+title Driving State
+
+package "Application" as appLayer {
+
+    class DrivingState <<control>> {
+        + {static} getInstance() : DrivingState
+        + entry() : void
+        + process(sm : StateMachine&) : void
+        + exit() : void
+    }
+
+    class ReadyState <<control>>
+
+    DrivingState .r.> ReadyState: <<use>>
+}
+
+package "Service" as serviceLayer {
+
+    class SimpleTimer <<service>> {
+        + start(duration : uint32_t) : void
+        + restart() : void
+        + stop() : void
+        + isTimeout() : bool
+    }    
+    class DifferentialDrive <<service>>
+}
+
+package "HAL" as hal {
+
+    package "Interfaces" as halInterfaces {
+        interface IDisplay {
+            + {abstract} clear() : void
+            + {abstract} gotoXY(xCoord : uint8_t, yCoord : uint8_t) : void
+            + {abstract} print(str : const String&) : size_t
+            + {abstract} print(str : const char[]) : size_t
+            + {abstract} print(value : uint8_t) : size_t
+            + {abstract} print(value : uint16_t) : size_t
+            + {abstract} print(value : uint32_t) : size_t
+            + {abstract} print(value : int8_t) : size_t
+            + {abstract} print(value : int16_t) : size_t
+            + {abstract} print(value : int32_t) : size_t
+        }
+
+        interface ILineSensors {
+            + {abstract} init() : void
+            + {abstract} calibrate() : void
+            + {abstract} readLine() : int16_t
+            + {abstract} getSensorValues() : const uint16_t*
+            + {abstract} isCalibrationSuccessful() : bool
+            + {abstract} getCalibErrorInfo() const : uint8_t
+            + {abstract} getNumLineSensors() const : uint8_t
+            + {abstract} getSensorValueMax() const : uint16_t
+        }
+
+        interface ILed {
+            + {abstract} enable(enableIt : bool) : void
+        }
+
+        interface IButton {
+            + {abstract} isPressed() : bool
+            + {abstract} waitForRelease() : void
+        }
+
+    }
+
+    class Board << namespace >> {
+        + getDisplay() : IDisplay&
+        + getLineSensors() : ILineSensors&
+        + getLedYellow() : ILed&
+        + getButtonA() : IButton&
+    }
+    class WebotsSerialDrv  {
+        + setRxChannel(channelId: int32_t) : void
+        + setTxChannel(channelId: int32_t ) : void
+        + print(str: const char[]) : void
+        + print(value: uint8_t ) : void
+        + print(value: uint16_t ) : void
+        + print(value: uint32_t ) : void
+        + print(value: int8_t ) : void
+        + print(value: int16_t ) : void
+        + print(value: int32_t ) : void
+        + println(str: const char[]) : void
+        + println(value: uint8_t ) : void
+        + println(value: uint16_t ) : void
+        + println(value: uint32_t ) : void
+        + println(value: int8_t ) : void
+        + println(value: int16_t ) : void
+        + println(value: int32_t ) : void
+        + write( buffer: const uint8_t*, length: size_t ) : size_t 
+        + readBytes( buffer: uint8_t*, length: size_t ) : size_t 
+    }
+}
+
+DrivingState *-> SimpleTimer
+DrivingState ..> DifferentialDrive: <<use>>
+DrivingState ...> IDisplay: <<use>>
+DrivingState ...> ILineSensors: <<use>>
+DrivingState ...> ILed: <<use>>
+DrivingState ...> Board: <<use>>
+DrivingState ...>WebotsSerialDrv: <<use>>
+DrivingState ...>IButton: <<use>>
+@enduml

doc/architecture/uml/LogicalView/ReinforcementLearning/ErrorState.plantuml

@@ -0,0 +1,52 @@
+@startuml
+
+title Error State
+
+package "Application" as appLayer {
+
+    class ErrorState <<control>> {
+        + {static} getInstance() : ErrorState
+        + entry() : void
+        + process(sm : StateMachine&) : void
+        + exit() : void
+        + setErrorMsg(msg : const String&) : void
+    }
+
+    class StartupState <<control>>
+
+    ErrorState .r.> StartupState: <<use>>
+}
+
+package "HAL" as hal {
+
+    package "Interfaces" as halInterfaces {
+        interface IDisplay {
+            + {abstract} clear() : void
+            + {abstract} gotoXY(xCoord : uint8_t, yCoord : uint8_t) : void
+            + {abstract} print(str : const String&) : size_t
+            + {abstract} print(str : const char[]) : size_t
+            + {abstract} print(value : uint8_t) : size_t
+            + {abstract} print(value : uint16_t) : size_t
+            + {abstract} print(value : uint32_t) : size_t
+            + {abstract} print(value : int8_t) : size_t
+            + {abstract} print(value : int16_t) : size_t
+            + {abstract} print(value : int32_t) : size_t
+        }
+
+        interface IButton {
+            + {abstract} isPressed() : bool
+            + {abstract} waitForRelease() : void
+        }
+    }
+
+    class Board << namespace >> {
+        + getDisplay() : IDisplay&
+        + getButtonA() : IButton&
+    }
+}
+
+ErrorState ..> Board: <<use>>
+ErrorState ..> IDisplay: <<use>>
+ErrorState ..> IButton: <<use>>
+
+@enduml

doc/architecture/uml/LogicalView/ReinforcementLearning/LineSensorsCalibrationState.plantuml

@@ -0,0 +1,91 @@
+@startuml
+
+title Line Sensors Calibration State
+
+package "Application" as appLayer {
+
+    class LineSensorsCalibrationState <<control>> {
+        + {static} getInstance() : LineSensorsCalibrationState
+        + entry() : void
+        + process(sm : StateMachine&) : void
+        + exit() : void
+    }
+
+    class ReadyState <<control>>
+    class ErrorState <<control>>
+
+    LineSensorsCalibrationState ..> ReadyState: <<use>>
+    LineSensorsCalibrationState ..> ErrorState: <<use>>
+}
+
+package "Service" as serviceLayer {
+
+    class SimpleTimer <<service>>
+    class RelativeEncoder <<service>>
+    class DifferentialDrive <<service>><<singleton>>
+    class Speedometer <<service>><<singleton>>
+
+    DifferentialDrive ..> Speedometer: <<use>>
+}
+
+package "HAL" as hal {
+
+    package "Interfaces" as halInterfaces {
+        interface IDisplay {
+            + {abstract} clear() : void
+            + {abstract} gotoXY(xCoord : uint8_t, yCoord : uint8_t) : void
+            + {abstract} print(str : const String&) : size_t
+            + {abstract} print(str : const char[]) : size_t
+            + {abstract} print(value : uint8_t) : size_t
+            + {abstract} print(value : uint16_t) : size_t
+            + {abstract} print(value : uint32_t) : size_t
+            + {abstract} print(value : int8_t) : size_t
+            + {abstract} print(value : int16_t) : size_t
+            + {abstract} print(value : int32_t) : size_t
+        }
+
+        interface IMotors {
+            + {abstract} setSpeeds(leftSpeed : int16_t, rightSpeed : int16_t) : void
+            + {abstract} getMaxSpeed() : int16_t
+        }
+
+        interface ILineSensors {
+            + {abstract} init() : void
+            + {abstract} calibrate() : void
+            + {abstract} readLine() : int16_t
+            + {abstract} getSensorValues() : const uint16_t*
+            + {abstract} getNumLineSensors() const : uint8_t
+            + {abstract} getSensorValueMax() const : uint16_t
+        }
+
+        interface IEncoders {
+            + {abstract} getCountsLeft() : int16_t
+            + {abstract} getCountsRight() : int16_t
+            + {abstract} getCountsAndResetLeft() : int16_t
+            + {abstract} getCountsAndResetRight() : int16_t
+            + {abstract} getResolution() const : uint16_t
+        }
+    }
+
+    class Board << namespace >> {
+        + getDisplay() : IDisplay&
+        + getMotors() : IMotors&
+        + getLineSensors() : ILineSensors&
+    }
+}
+
+appLayer -[hidden]-- serviceLayer
+serviceLayer -[hidden]-- hal
+
+LineSensorsCalibrationState ....> IDisplay: <<use>>
+LineSensorsCalibrationState ....> ILineSensors: <<use>>
+LineSensorsCalibrationState ....> Board: <<use>>
+LineSensorsCalibrationState *--> SimpleTimer
+LineSensorsCalibrationState *--> RelativeEncoder
+LineSensorsCalibrationState ...> DifferentialDrive: <<use>>
+
+DifferentialDrive ...> IMotors: <<use>>
+Speedometer ..> IEncoders: <<use>>
+Speedometer ..> IMotors: <<use>>
+
+@enduml