From db016f8b3fdf4555c76bc986832f2e5255c213b4 Mon Sep 17 00:00:00 2001 From: Piotr Gawlowicz Date: Tue, 18 Jan 2022 19:51:37 +0100 Subject: [PATCH] update --- examples/multi-agent/agent1.py | 56 ++++++++ examples/multi-agent/agent2.py | 56 ++++++++ examples/multi-agent/mygym.cc | 223 ++++++++++++++++++++++++++++++ examples/multi-agent/mygym.h | 56 ++++++++ examples/multi-agent/sim.cc | 83 +++++++++++ examples/opengym-2/mygym.cc | 22 +-- examples/opengym-2/simple_test.py | 4 + examples/opengym-2/test.py | 21 +-- model/opengym_interface.cc | 16 +-- test/opengym-test-suite.cc | 2 +- 10 files changed, 504 insertions(+), 35 deletions(-) create mode 100755 examples/multi-agent/agent1.py create mode 100755 examples/multi-agent/agent2.py create mode 100644 examples/multi-agent/mygym.cc create mode 100644 examples/multi-agent/mygym.h create mode 100644 examples/multi-agent/sim.cc diff --git a/examples/multi-agent/agent1.py b/examples/multi-agent/agent1.py new file mode 100755 index 0000000000..5cb88141c1 --- /dev/null +++ b/examples/multi-agent/agent1.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from ns3gym import ns3env + +__author__ = "Piotr Gawlowicz" +__copyright__ = "Copyright (c) 2020, Technische Universität Berlin" +__version__ = "0.1.0" +__email__ = "gawlowicz@tkn.tu-berlin.de" + + +port = 5555 +env = ns3env.Ns3Env(port=port, startSim=False) +env.reset() + +ob_space = env.observation_space +ac_space = env.action_space +print("Observation space: ", ob_space, ob_space.dtype) +print("Action space: ", ac_space, ac_space.dtype) + + +stepIdx = 0 +currIt = 0 +iterationNum = 3 + +try: + while True: + obs = env.reset() + print("Step: ", stepIdx) + print("---obs: ", obs) + + while True: + stepIdx += 1 + action = env.action_space.sample() + print("---action: ", action) + + print("Step: ", stepIdx) + obs, reward, done, info = env.step(action) + print("---obs, reward, done, info: ", obs, reward, done, info) + + input("press enter....") + + if done: + break + + currIt += 1 + if currIt == iterationNum: + break + + +except KeyboardInterrupt: + print("Ctrl-C -> Exit") +finally: + env.close() + print("Done") \ No newline at end of file diff --git a/examples/multi-agent/agent2.py b/examples/multi-agent/agent2.py new file mode 100755 index 0000000000..42c184234d --- /dev/null +++ b/examples/multi-agent/agent2.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from ns3gym import ns3env + +__author__ = "Piotr Gawlowicz" +__copyright__ = "Copyright (c) 2020, Technische Universität Berlin" +__version__ = "0.1.0" +__email__ = "gawlowicz@tkn.tu-berlin.de" + + +port = 5556 +env = ns3env.Ns3Env(port=port, startSim=False) +env.reset() + +ob_space = env.observation_space +ac_space = env.action_space +print("Observation space: ", ob_space, ob_space.dtype) +print("Action space: ", ac_space, ac_space.dtype) + + +stepIdx = 0 +currIt = 0 +iterationNum = 3 + +try: + while True: + obs = env.reset() + print("Step: ", stepIdx) + print("---obs: ", obs) + + while True: + stepIdx += 1 + action = env.action_space.sample() + print("---action: ", action) + + print("Step: ", stepIdx) + obs, reward, done, info = env.step(action) + print("---obs, reward, done, info: ", obs, reward, done, info) + + input("press enter....") + + if done: + break + + currIt += 1 + if currIt == iterationNum: + break + + +except KeyboardInterrupt: + print("Ctrl-C -> Exit") +finally: + env.close() + print("Done") \ No newline at end of file diff --git a/examples/multi-agent/mygym.cc b/examples/multi-agent/mygym.cc new file mode 100644 index 0000000000..282e97c7e8 --- /dev/null +++ b/examples/multi-agent/mygym.cc @@ -0,0 +1,223 @@ +/* -*- Mode: C++; c-file-style: "gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 Technische Universität Berlin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Piotr Gawlowicz + */ + +#include "mygym.h" +#include "ns3/object.h" +#include "ns3/core-module.h" +#include "ns3/wifi-module.h" +#include "ns3/node-list.h" +#include "ns3/log.h" +#include +#include + +namespace ns3 { + +NS_LOG_COMPONENT_DEFINE ("MyGymEnv"); + +NS_OBJECT_ENSURE_REGISTERED (MyGymEnv); + +MyGymEnv::MyGymEnv () +{ + NS_LOG_FUNCTION (this); + m_interval = Seconds(0.1); + + Simulator::Schedule (Seconds(0.0), &MyGymEnv::ScheduleNextStateRead, this); +} + +MyGymEnv::MyGymEnv (uint32_t id, Time stepTime) +{ + NS_LOG_FUNCTION (this); + m_agentId = id; + m_interval = stepTime; + + Simulator::Schedule (Seconds(0.0), &MyGymEnv::ScheduleNextStateRead, this); +} + +void +MyGymEnv::ScheduleNextStateRead () +{ + NS_LOG_FUNCTION (this); + Simulator::Schedule (m_interval, &MyGymEnv::ScheduleNextStateRead, this); + Notify(); +} + +MyGymEnv::~MyGymEnv () +{ + NS_LOG_FUNCTION (this); +} + +TypeId +MyGymEnv::GetTypeId (void) +{ + static TypeId tid = TypeId ("MyGymEnv") + .SetParent () + .SetGroupName ("OpenGym") + .AddConstructor () + ; + return tid; +} + +void +MyGymEnv::DoDispose () +{ + NS_LOG_FUNCTION (this); +} + +/* +Define observation space +*/ +Ptr +MyGymEnv::GetObservationSpace() +{ + uint32_t nodeNum = 5; + float low = 0.0; + float high = 10.0; + std::vector shape = {nodeNum,}; + std::string dtype = TypeNameGet (); + + Ptr discrete = CreateObject (nodeNum); + Ptr box = CreateObject (low, high, shape, dtype); + + Ptr space = CreateObject (); + space->Add("box", box); + space->Add("discrete", discrete); + + NS_LOG_UNCOND ("AgendId: "<< m_agentId << " MyGetObservationSpace: " << space); + return space; +} + +/* +Define action space +*/ +Ptr +MyGymEnv::GetActionSpace() +{ + uint32_t nodeNum = 5; + float low = 0.0; + float high = 10.0; + std::vector shape = {nodeNum,}; + std::string dtype = TypeNameGet (); + + Ptr discrete = CreateObject (nodeNum); + Ptr box = CreateObject (low, high, shape, dtype); + + Ptr space = CreateObject (); + space->Add("box", box); + space->Add("discrete", discrete); + + NS_LOG_UNCOND ("AgendId: "<< m_agentId << " MyGetActionSpace: " << space); + return space; +} + +/* +Define game over condition +*/ +bool +MyGymEnv::GetGameOver() +{ + bool isGameOver = false; + bool test = false; + static float stepCounter = 0.0; + stepCounter += 1; + if (stepCounter == 10 && test) { + isGameOver = true; + } + NS_LOG_UNCOND ("AgendId: "<< m_agentId << " MyGetGameOver: " << isGameOver); + return isGameOver; +} + +/* +Collect observations +*/ +Ptr +MyGymEnv::GetObservation() +{ + uint32_t nodeNum = 5; + uint32_t low = 0.0; + uint32_t high = 10.0; + Ptr rngInt = CreateObject (); + + std::vector shape = {nodeNum,}; + Ptr > box = CreateObject >(shape); + + // generate random data + for (uint32_t i = 0; iGetInteger(low, high); + box->AddValue(value); + } + + Ptr discrete = CreateObject(nodeNum); + uint32_t value = rngInt->GetInteger(low, high); + discrete->SetValue(value); + + Ptr data = CreateObject (); + data->Add(box); + data->Add(discrete); + + // Print data from tuple + Ptr > mbox = DynamicCast >(data->Get(0)); + Ptr mdiscrete = DynamicCast(data->Get(1)); + NS_LOG_UNCOND ("AgendId: "<< m_agentId << " MyGetObservation: " << data); + NS_LOG_UNCOND ("---" << mbox); + NS_LOG_UNCOND ("---" << mdiscrete); + + return data; +} + +/* +Define reward function +*/ +float +MyGymEnv::GetReward() +{ + static float reward = 0.0; + reward += 1; + return reward; +} + +/* +Define extra info. Optional +*/ +std::string +MyGymEnv::GetExtraInfo() +{ + std::string myInfo = "testInfo"; + myInfo += "|123"; + NS_LOG_UNCOND("AgendId: "<< m_agentId << " MyGetExtraInfo: " << myInfo); + return myInfo; +} + +/* +Execute received actions +*/ +bool +MyGymEnv::ExecuteActions(Ptr action) +{ + Ptr dict = DynamicCast(action); + Ptr > box = DynamicCast >(dict->Get("box")); + Ptr discrete = DynamicCast(dict->Get("discrete")); + + NS_LOG_UNCOND ("AgendId: "<< m_agentId << " MyExecuteActions: " << action); + NS_LOG_UNCOND ("---" << box); + NS_LOG_UNCOND ("---" << discrete); + return true; +} + +} // ns3 namespace \ No newline at end of file diff --git a/examples/multi-agent/mygym.h b/examples/multi-agent/mygym.h new file mode 100644 index 0000000000..53b0e7a107 --- /dev/null +++ b/examples/multi-agent/mygym.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; c-file-style: "gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 Technische Universität Berlin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Piotr Gawlowicz + */ + + +#ifndef MY_GYM_ENTITY_H +#define MY_GYM_ENTITY_H + +#include "ns3/opengym-module.h" + +namespace ns3 { + +class MyGymEnv : public OpenGymEnv +{ +public: + MyGymEnv (); + MyGymEnv (uint32_t id, Time stepTime); + virtual ~MyGymEnv (); + static TypeId GetTypeId (void); + virtual void DoDispose (); + + Ptr GetActionSpace(); + Ptr GetObservationSpace(); + bool GetGameOver(); + Ptr GetObservation(); + float GetReward(); + std::string GetExtraInfo(); + bool ExecuteActions(Ptr action); + +private: + void ScheduleNextStateRead(); + + Time m_interval; + uint32_t m_agentId; +}; + +} + + +#endif // MY_GYM_ENTITY_H diff --git a/examples/multi-agent/sim.cc b/examples/multi-agent/sim.cc new file mode 100644 index 0000000000..9749cabc72 --- /dev/null +++ b/examples/multi-agent/sim.cc @@ -0,0 +1,83 @@ +/* -*- Mode: C++; c-file-style: "gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 Piotr Gawlowicz + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Piotr Gawlowicz + * + */ + +#include "ns3/core-module.h" +#include "ns3/opengym-module.h" +#include "mygym.h" + +using namespace ns3; + +NS_LOG_COMPONENT_DEFINE ("OpenGym"); + +int +main (int argc, char *argv[]) +{ + // Parameters of the scenario + uint32_t simSeed = 1; + double simulationTime = 1; //seconds + double envStepTime = 0.1; //seconds, ns3gym env step time interval + uint32_t openGymPort = 5555; + uint32_t testArg = 0; + + CommandLine cmd; + // required parameters for OpenGym interface + cmd.AddValue ("openGymPort", "Port number for OpenGym env. Default: 5555", openGymPort); + cmd.AddValue ("simSeed", "Seed for random generator. Default: 1", simSeed); + // optional parameters + cmd.AddValue ("simTime", "Simulation time in seconds. Default: 10s", simulationTime); + cmd.AddValue ("stepTime", "Gym Env step time in seconds. Default: 0.1s", envStepTime); + cmd.AddValue ("testArg", "Extra simulation argument. Default: 0", testArg); + cmd.Parse (argc, argv); + + NS_LOG_UNCOND("Ns3Env parameters:"); + NS_LOG_UNCOND("--simulationTime: " << simulationTime); + NS_LOG_UNCOND("--openGymPort: " << openGymPort); + NS_LOG_UNCOND("--envStepTime: " << envStepTime); + NS_LOG_UNCOND("--seed: " << simSeed); + NS_LOG_UNCOND("--testArg: " << testArg); + + RngSeedManager::SetSeed (1); + RngSeedManager::SetRun (simSeed); + + // OpenGym Env for agent 1 + uint32_t agentId = 1; + openGymPort = 5555; + Ptr openGymInterface1 = CreateObject (openGymPort); + Ptr myGymEnv1 = CreateObject (agentId, Seconds(envStepTime)); + myGymEnv1->SetOpenGymInterface(openGymInterface1); + + // OpenGym Env for agent 2 + agentId = 2; + openGymPort = 5556; + Ptr openGymInterface2 = CreateObject (openGymPort); + Ptr myGymEnv2 = CreateObject (agentId, Seconds(envStepTime)); + myGymEnv2->SetOpenGymInterface(openGymInterface2); + + NS_LOG_UNCOND ("Simulation start"); + Simulator::Stop (Seconds (simulationTime)); + Simulator::Run (); + NS_LOG_UNCOND ("Simulation stop"); + + openGymInterface1->NotifySimulationEnd(); + openGymInterface2->NotifySimulationEnd(); + Simulator::Destroy (); + +} diff --git a/examples/opengym-2/mygym.cc b/examples/opengym-2/mygym.cc index 15dc2f0226..188910c157 100644 --- a/examples/opengym-2/mygym.cc +++ b/examples/opengym-2/mygym.cc @@ -95,8 +95,8 @@ MyGymEnv::GetObservationSpace() Ptr box = CreateObject (low, high, shape, dtype); Ptr space = CreateObject (); - space->Add("box", box); - space->Add("discrete", discrete); + space->Add("myVector", box); + space->Add("myValue", discrete); NS_LOG_UNCOND ("MyGetObservationSpace: " << space); return space; @@ -118,8 +118,8 @@ MyGymEnv::GetActionSpace() Ptr box = CreateObject (low, high, shape, dtype); Ptr space = CreateObject (); - space->Add("box", box); - space->Add("discrete", discrete); + space->Add("myActionVector", box); + space->Add("myActionValue", discrete); NS_LOG_UNCOND ("MyGetActionSpace: " << space); return space; @@ -166,13 +166,13 @@ MyGymEnv::GetObservation() uint32_t value = rngInt->GetInteger(low, high); discrete->SetValue(value); - Ptr data = CreateObject (); - data->Add(box); - data->Add(discrete); + Ptr data = CreateObject (); + data->Add("myVector",box); + data->Add("myValue",discrete); // Print data from tuple - Ptr > mbox = DynamicCast >(data->Get(0)); - Ptr mdiscrete = DynamicCast(data->Get(1)); + Ptr > mbox = DynamicCast >(data->Get("myVector")); + Ptr mdiscrete = DynamicCast(data->Get("myValue")); NS_LOG_UNCOND ("MyGetObservation: " << data); NS_LOG_UNCOND ("---" << mbox); NS_LOG_UNCOND ("---" << mdiscrete); @@ -210,8 +210,8 @@ bool MyGymEnv::ExecuteActions(Ptr action) { Ptr dict = DynamicCast(action); - Ptr > box = DynamicCast >(dict->Get("box")); - Ptr discrete = DynamicCast(dict->Get("discrete")); + Ptr > box = DynamicCast >(dict->Get("myActionVector")); + Ptr discrete = DynamicCast(dict->Get("myActionValue")); NS_LOG_UNCOND ("MyExecuteActions: " << action); NS_LOG_UNCOND ("---" << box); diff --git a/examples/opengym-2/simple_test.py b/examples/opengym-2/simple_test.py index 72776f806c..91aa3b467e 100755 --- a/examples/opengym-2/simple_test.py +++ b/examples/opengym-2/simple_test.py @@ -35,6 +35,10 @@ print("Step: ", stepIdx) print("---obs, reward, done, info: ", obs, reward, done, info) + myVector = obs["myVector"] + myValue = obs["myValue"] + print("---myVector: ", myVector) + print("---myValue: ", myValue) if done: break diff --git a/examples/opengym-2/test.py b/examples/opengym-2/test.py index 71bff2e97a..42966caca3 100755 --- a/examples/opengym-2/test.py +++ b/examples/opengym-2/test.py @@ -10,19 +10,8 @@ __email__ = "gawlowicz@tkn.tu-berlin.de" -parser = argparse.ArgumentParser(description='Start simulation script on/off') -parser.add_argument('--start', - type=int, - default=1, - help='Start ns-3 simulation script 0/1, Default: 1') -parser.add_argument('--iterations', - type=int, - default=1, - help='Number of iterations, Default: 1') -args = parser.parse_args() -startSim = bool(args.start) -iterationNum = int(args.iterations) - +startSim = False +iterationNum = 1 port = 5555 simTime = 5 # seconds stepTime = 0.5 # seconds @@ -33,8 +22,6 @@ debug = False env = ns3env.Ns3Env(port=port, stepTime=stepTime, startSim=startSim, simSeed=seed, simArgs=simArgs, debug=debug) -# simpler: -#env = ns3env.Ns3Env() env.reset() ob_space = env.observation_space @@ -60,6 +47,10 @@ print("Step: ", stepIdx) obs, reward, done, info = env.step(action) print("---obs, reward, done, info: ", obs, reward, done, info) + myVector = obs["myVector"] + myValue = obs["myValue"] + print("---myVector: ", myVector) + print("---myValue: ", myValue) if done: stepIdx = 0 diff --git a/model/opengym_interface.cc b/model/opengym_interface.cc index 04f02c2967..24196ee03a 100644 --- a/model/opengym_interface.cc +++ b/model/opengym_interface.cc @@ -187,14 +187,14 @@ OpenGymInterface::Init() } // send init msg to python - zmq::message_t request(simInitMsg.ByteSize());; - simInitMsg.SerializeToArray(request.data(), simInitMsg.ByteSize()); - m_zmq_socket.send (request); + zmq::message_t request(simInitMsg.ByteSizeLong());; + simInitMsg.SerializeToArray(request.data(), simInitMsg.ByteSizeLong()); + m_zmq_socket.send (request, zmq::send_flags::none); // receive init ack msg form python ns3opengym::SimInitAck simInitAck; zmq::message_t reply; - m_zmq_socket.recv (&reply); + (void) m_zmq_socket.recv (reply, zmq::recv_flags::none); simInitAck.ParseFromArray(reply.data(), reply.size()); bool done = simInitAck.done(); @@ -254,14 +254,14 @@ OpenGymInterface::NotifyCurrentState() envStateMsg.set_info(extraInfo); // send env state msg to python - zmq::message_t request(envStateMsg.ByteSize());; - envStateMsg.SerializeToArray(request.data(), envStateMsg.ByteSize()); - m_zmq_socket.send (request); + zmq::message_t request(envStateMsg.ByteSizeLong());; + envStateMsg.SerializeToArray(request.data(), envStateMsg.ByteSizeLong()); + m_zmq_socket.send (request, zmq::send_flags::none); // receive act msg form python ns3opengym::EnvActMsg envActMsg; zmq::message_t reply; - m_zmq_socket.recv (&reply); + (void) m_zmq_socket.recv (reply, zmq::recv_flags::none); envActMsg.ParseFromArray(reply.data(), reply.size()); if (m_simEnd) { diff --git a/test/opengym-test-suite.cc b/test/opengym-test-suite.cc index 1ba4d8852e..45fb5be624 100644 --- a/test/opengym-test-suite.cc +++ b/test/opengym-test-suite.cc @@ -1,7 +1,7 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ // Include a header file from your module to test. -#include "ns3/opengym-module.h" +//#include "ns3/opengym-module.h" // An essential include is test.h #include "ns3/test.h"