feat: add clock versions (#35)

- switch and checkers has v2 and v3 version with clock info.

Author: koulanurag

README.md

@@ -1,8 +1,10 @@
 # ma-gym
-A collection of multi agent environments based on OpenAI gym.
+It's a collection of multi agent environments based on OpenAI gym. Also, you can use [**minimal-marl**](https://github.com/koulanurag/minimal-marl) to warm-start training of agents.
 
 ![Python package](https://github.com/koulanurag/ma-gym/workflows/Python%20package/badge.svg) 
 ![Upload Python Package](https://github.com/koulanurag/ma-gym/workflows/Upload%20Python%20Package/badge.svg)
+[![Wiki Docs](https://img.shields.io/badge/-Wiki%20Docs-informational?style=flat)](https://github.com/koulanurag/ma-gym/wiki)
+
 
 ## Installation
 Using PyPI:
@@ -45,7 +47,7 @@ while not all(done_n):
 env.close()
 ```
 
-Please refer to [Wiki](https://github.com/koulanurag/ma-gym/wiki/Usage) for complete usage details
+Please refer to [**Wiki**](https://github.com/koulanurag/ma-gym/wiki/Usage) for complete usage details
 
 ## Environments:
 - [x] Checkers

ma_gym/__init__.py

@@ -17,28 +17,23 @@
 
 # add new environments : iterate over full observability
 for i, observability in enumerate([False, True]):
-    register(
-        id='CrossOver-v' + str(i),
-        entry_point='ma_gym.envs.crossover:CrossOver',
-        kwargs={'full_observable': observability, 'step_cost': -0.5}
-    )
-
-    register(
-        id='Checkers-v' + str(i),
-        entry_point='ma_gym.envs.checkers:Checkers',
-        kwargs={'full_observable': observability}
-    )
 
-    register(
-        id='Switch2-v' + str(i),
-        entry_point='ma_gym.envs.switch:Switch',
-        kwargs={'n_agents': 2, 'full_observable': observability, 'step_cost': -0.1}
-    )
-    register(
-        id='Switch4-v' + str(i),
-        entry_point='ma_gym.envs.switch:Switch',
-        kwargs={'n_agents': 4, 'full_observable': observability, 'step_cost': -0.1}
-    )
+    for clock in [False, True]:
+        register(
+            id='Checkers-v{}'.format(i + (2 if clock else 0)),
+            entry_point='ma_gym.envs.checkers:Checkers',
+            kwargs={'full_observable': observability, 'step_cost': -0.01, 'clock': clock}
+        )
+        register(
+            id='Switch2-v{}'.format(i + (2 if clock else 0)),
+            entry_point='ma_gym.envs.switch:Switch',
+            kwargs={'n_agents': 2, 'full_observable': observability, 'step_cost': -0.1, 'clock': clock}
+        )
+        register(
+            id='Switch4-v{}'.format(i + (2 if clock else 0)),
+            entry_point='ma_gym.envs.switch:Switch',
+            kwargs={'n_agents': 4, 'full_observable': observability, 'step_cost': -0.1, 'clock': clock}
+        )
 
     for num_max_cars in [4, 10]:
         register(

ma_gym/envs/checkers/checkers.py

@@ -26,17 +26,18 @@ class Checkers(gym.Env):
     """
     metadata = {'render.modes': ['human', 'rgb_array']}
 
-    def __init__(self, full_observable=False, step_cost=-0.01, max_steps=100):
+    def __init__(self, full_observable=False, step_cost=-0.01, max_steps=100, clock=False):
         self._grid_shape = (3, 8)
         self.n_agents = 2
         self._max_steps = max_steps
         self._step_count = None
         self._step_cost = step_cost
         self.full_observable = full_observable
+        self._add_clock = clock
 
         self.action_space = MultiAgentActionSpace([spaces.Discrete(5) for _ in range(self.n_agents)])
-        self._obs_high = np.ones(2 + (3 * 3 * 5))
-        self._obs_low = np.zeros(2 + (3 * 3 * 5))
+        self._obs_high = np.ones(2 + (3 * 3 * 5) + (1 if clock else 0))
+        self._obs_low = np.zeros(2 + (3 * 3 * 5) + (1 if clock else 0))
         if self.full_observable:
             self._obs_high = np.tile(self._obs_high, self.n_agents)
             self._obs_low = np.tile(self._obs_low, self.n_agents)
@@ -124,7 +125,8 @@ def get_agent_obs(self):
             _agent_i_obs += _agent_i_neighbour.flatten().tolist()
 
             # adding time
-            # _agent_i_obs += [self._step_count / self._max_steps]
+            if self._add_clock:
+                _agent_i_obs += [self._step_count / self._max_steps]
             _obs.append(_agent_i_obs)
 
         if self.full_observable:

ma_gym/envs/switch/switch_one_corridor.py

@@ -16,14 +16,16 @@
 class Switch(gym.Env):
     metadata = {'render.modes': ['human', 'rgb_array']}
 
-    def __init__(self, full_observable: bool = False, step_cost: float = 0, n_agents: int = 4, max_steps: int = 50):
+    def __init__(self, full_observable: bool = False, step_cost: float = 0, n_agents: int = 4, max_steps: int = 50,
+                 clock: bool = True):
         assert 2 <= n_agents <= 4, 'Number of Agents has to be in range [2,4]'
         self._grid_shape = (3, 7)
         self.n_agents = n_agents
         self._max_steps = max_steps
         self._step_count = None
         self._step_cost = step_cost
         self._total_episode_reward = None
+        self._add_clock = clock
 
         self.action_space = MultiAgentActionSpace([spaces.Discrete(5) for _ in range(self.n_agents)])  # l,r,t,d,noop
 
@@ -44,8 +46,8 @@ def __init__(self, full_observable: bool = False, step_cost: float = 0, n_agents
 
         self.full_observable = full_observable
         # agent pos (2)
-        self._obs_high = np.array([1., 1.], dtype=np.float32)
-        self._obs_low = np.array([0., 0.], dtype=np.float32)
+        self._obs_high = np.ones(2 + (1 if self._add_clock else 0))
+        self._obs_low = np.zeros(2 + (1 if self._add_clock else 0))
         if self.full_observable:
             self._obs_high = np.tile(self._obs_high, self.n_agents)
             self._obs_low = np.tile(self._obs_low, self.n_agents)
@@ -91,7 +93,8 @@ def get_agent_obs(self):
             pos = self.agent_pos[agent_i]
             _agent_i_obs = [round(pos[0] / (self._grid_shape[0] - 1), 2),
                             round(pos[1] / (self._grid_shape[1] - 1), 2)]
-            # _agent_i_obs += [self._step_count / self._max_steps]  # add current step count (for time reference)
+            if self._add_clock:
+                _agent_i_obs += [self._step_count / self._max_steps]  # add current step count (for time reference)
             _obs.append(_agent_i_obs)
 
         if self.full_observable:

setup.py

@@ -12,7 +12,7 @@
 extras['all'] = [item for group in extras.values() for item in group]
 
 setup(name='ma_gym',
-      version='0.0.6',
+      version='0.0.7',
       description='A collection of multi agent environments based on OpenAI gym.',
       long_description_content_type='text/markdown',
       long_description=open(path.join(path.abspath(path.dirname(__file__)), 'README.md'), encoding='utf-8').read(),

tests/envs/test_switch2.py

@@ -125,3 +125,21 @@ def test_optimal_rollout(env):
                                                                                                reward_n, step_i)
             assert done == target_dones[step_i]
             step_i += 1
+
+
+@parametrize_plus('env',
+                  [fixture_ref(env),
+                   fixture_ref(env_full)])
+def test_max_steps(env):
+    """ All agent remain at their initial position for the entire duration"""
+    for _ in range(2):
+        env.reset()
+        step_i = 0
+        done = [False for _ in range(env.n_agents)]
+        while not all(done):
+            obs, reward_n, done, _ = env.step([4 for _ in range(env.n_agents)])
+            target_reward = [env._step_cost for _ in range(env.n_agents)]
+            step_i += 1
+            assert (reward_n == target_reward), \
+                'step_cost is not correct. Expected {} ; Got {}'.format(target_reward, reward_n)
+        assert step_i == env._max_steps, 'max-steps should be reached'