143 Retry for ml tests. (#146)

* Add pytest-retry enabling retries for flapping ml tests.

* Run notebooks and examples with all supported Python versions.

Signed-off-by: chenkins <[email protected]>

* Fix notebooks require redis - run them in separate tox to enhance parallelization in gh.

Signed-off-by: chenkins <[email protected]>

---------

Signed-off-by: chenkins <[email protected]>

Author: chenkins

.github/workflows/checks.yml

@@ -52,12 +52,16 @@ jobs:
 
     notebooks:
         runs-on: ubuntu-22.04
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: [ "3.10", "3.11", "3.12" ]
         steps:
             -   uses: actions/checkout@v4
             -   name: Set up Python ${{ matrix.python-version }}
                 uses: actions/setup-python@v5
                 with:
-                    python-version: "3.10"
+                    python-version: ${{ matrix.python-version }}
             -   name: Start Redis
                 uses: supercharge/[email protected]
                 with:
@@ -72,12 +76,16 @@ jobs:
 
     examples:
         runs-on: ubuntu-22.04
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: [ "3.10", "3.11", "3.12" ]
         steps:
             -   uses: actions/checkout@v4
             -   name: Set up Python ${{ matrix.python-version }}
                 uses: actions/setup-python@v5
                 with:
-                    python-version: "3.10"
+                    python-version: ${{ matrix.python-version }}
             -   name: Install tox
                 run: |
                     python -m pip install --upgrade pip

.github/workflows/main.yml

@@ -54,12 +54,16 @@ jobs:
 
     notebooks:
         runs-on: ubuntu-22.04
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: [ "3.10", "3.11", "3.12" ]
         steps:
             -   uses: actions/checkout@v4
             -   name: Set up Python ${{ matrix.python-version }}
                 uses: actions/setup-python@v5
                 with:
-                    python-version: "3.10"
+                    python-version: ${{ matrix.python-version }}
             -   name: Start Redis
                 uses: supercharge/[email protected]
                 with:
@@ -74,12 +78,16 @@ jobs:
 
     examples:
         runs-on: ubuntu-22.04
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version: [ "3.10", "3.11", "3.12" ]
         steps:
             -   uses: actions/checkout@v4
-            -   name: Set up Python
+            -   name: Set up Python ${{ matrix.python-version }}
                 uses: actions/setup-python@v5
                 with:
-                    python-version: "3.10"
+                    python-version: ${{ matrix.python-version }}
             -   name: Install tox
                 run: |
                     python -m pip install --upgrade pip

pyproject.toml

@@ -65,6 +65,7 @@ dev = [
     "numpydoc",
     "pip-tools",
     "pytest",
+    "pytest-retry",
     "snakeviz",
     "tox",
     "twine",

requirements-dev.txt

@@ -438,6 +438,9 @@ pytest==8.3.4
     # via
     #   flatland-rl (pyproject.toml)
     #   nbmake
+    #   pytest-retry
+pytest-retry==1.7.0
+    # via flatland-rl (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via
     #   arrow

requirements.txt

@@ -177,3 +177,6 @@ wcwidth==0.2.13
     # via prompt-toolkit
 widgetsnbextension==4.0.13
     # via ipywidgets
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-env_list = py{310,311,312},notebooks,py{310,311,312}-ml,py{310,311,312}-verify-install
+env_list = py{310,311,312},py{310,311,312}-ml,py{310,311,312}-verify-install
 
 [gh-actions]
 python =
@@ -38,6 +38,7 @@ deps =
 set_env =
     BENCHMARK_EPISODES_FOLDER = {tox_root}/episodes
 commands =
+    python --version
     python -m pytest {posargs} --ignore=tests/ml
 
 [testenv:py{310,311,312}-ml]
@@ -49,7 +50,7 @@ deps =
     -r requirements-ml.txt
 commands =
     python --version
-    python -m pytest {posargs}
+    python -m pytest --retries 2 --retry-delay 5 {posargs}
 
 [testenv:lint]
 base_python = python3.10
@@ -83,22 +84,24 @@ set_env =
 commands =
     python benchmarks/profile_all_examples.py
 
-[testenv:examples]
-base_python = python3.10
+[testenv:py{310,311,312}-examples]
+platform = linux|linux2|darwin
 deps =
     -r requirements-dev.txt
 commands =
+    python --version
     python benchmarks/run_all_examples.py
 
-[testenv:notebooks]
-base_python = python3.10
+[testenv:py{310,311,312}-notebooks]
+platform = linux|linux2|darwin
 allowlist_externals =
     bash
     pwd
 deps =
     -r requirements-dev.txt
 commands =
-    python -m pytest --nbmake  \
+    python --version
+    python -m pytest -s --nbmake  \
         notebooks/simple-graph-plot-2022.ipynb \
         notebooks/Agent-Close-Following.ipynb \
         notebooks/simple_example_manual_control.ipynb \