Write benchmarking and profiling outputs to files and upload artifact in gh-action.

Author: chenkins

.github/workflows/main.yml

@@ -112,6 +112,10 @@ jobs:
                     pip install tox tox-gh-actions
             -   name: Run profiling
                 run: tox run -e profiling
+            -   uses: actions/upload-artifact@v4
+                with:
+                    name: upload-benchmarking-results
+                    path: ${GITHUB_WORKSPACE}/.tox/profiling/log
 
     benchmarks:
         runs-on: ubuntu-22.04
@@ -127,6 +131,10 @@ jobs:
                     pip install tox tox-gh-actions
             -   name: Run benchmarks
                 run: tox run -e benchmarks
+            -   uses: actions/upload-artifact@v4
+                with:
+                    name: upload-benchmarking-results
+                    path: ${GITHUB_WORKSPACE}/.tox/benchmarks/log
 
     publish:
         if: ${{ false }}  # disabled until decided when it should run exactly

benchmarks/benchmark_all_examples.py

@@ -1,3 +1,4 @@
+import os
 import runpy
 import sys
 from io import StringIO
@@ -10,30 +11,37 @@
 
 from benchmarks.benchmark_utils import swap_attr
 
-for entry in [entry for entry in importlib_resources.contents('examples') if
-              not pkg_resources.resource_isdir('examples', entry)
-              and entry.endswith(".py")
-              and '__init__' not in entry
-              and 'DELETE' not in entry
-              ]:
-    print("*****************************************************************")
-    print("Benchmarking {}".format(entry))
-    print("*****************************************************************")
+if __name__ == '__main__':
 
-    with path('examples', entry) as file_in:
-        with Benchmarker(cycle=20, extra=1) as bench:
-            @bench(entry)
-            def _(_):
-                # prevent Benchmarker from doing "ZeroDivisionError: float division by zero:
-                #    ratio = base_time / real_time"
-                sleep(0.001)
-                # In order to pipe input into examples that have input(),
-                # we use the test package, which is meant for internal use by Python only internal and
-                # Any use of this package outside of Python’s standard library is discouraged as code (..)
-                # can change or be removed without notice between releases of Python.
-                # https://docs.python.org/3/library/test.html
-                # TODO remove input() from examples?
-                with swap_attr(sys, "stdin", StringIO("q")):
-                    runpy.run_path(file_in, run_name="__main__", init_globals={
-                    'argv': ['--sleep-for-animation=False', '--do_rendering=False']
-                })
+    for entry in [entry for entry in importlib_resources.contents('examples') if
+                  not pkg_resources.resource_isdir('examples', entry)
+                  and entry.endswith(".py")
+                  and '__init__' not in entry
+                  and 'DELETE' not in entry
+                  ]:
+        print("*****************************************************************")
+        print("Benchmarking {}".format(entry))
+        print("*****************************************************************")
+
+        benchmarks_output_folder = os.environ.get('BENCHMARKS_OUTPUT_FOLDER', None)
+        outfile = None
+        if benchmarks_output_folder:
+            outfile = os.path.join(benchmarks_output_folder, f"{entry}.json")
+
+        with path('examples', entry) as file_in:
+            with Benchmarker(cycle=20, extra=1, outfile=outfile) as bench:
+                @bench(entry)
+                def _(_):
+                    # prevent Benchmarker from doing "ZeroDivisionError: float division by zero:
+                    #    ratio = base_time / real_time"
+                    sleep(0.001)
+                    # In order to pipe input into examples that have input(),
+                    # we use the test package, which is meant for internal use by Python only internal and
+                    # Any use of this package outside of Python’s standard library is discouraged as code (..)
+                    # can change or be removed without notice between releases of Python.
+                    # https://docs.python.org/3/library/test.html
+                    # TODO remove input() from examples?
+                    with swap_attr(sys, "stdin", StringIO("q")):
+                        runpy.run_path(file_in, run_name="__main__", init_globals={
+                            'argv': ['--sleep-for-animation=False', '--do_rendering=False']
+                        })

benchmarks/profile_all_examples.py

@@ -1,6 +1,8 @@
 import cProfile
+import os
 import runpy
 import sys
+from fileinput import filename
 from io import StringIO
 
 import importlib_resources
@@ -16,14 +18,19 @@ def profile(resource, entry):
         print("*****************************************************************")
         print("Profiling {}".format(entry))
         print("*****************************************************************")
+        profiling_output_folder = os.environ.get('PROFILING_OUTPUT_FOLDER', None)
+        outfile = None
+        if profiling_output_folder:
+            outfile = os.path.join(profiling_output_folder, f"{entry}.prof")
+
         with swap_attr(sys, "stdin", StringIO("q")):
             global my_func
 
             def my_func(): runpy.run_path(file_in, run_name="__main__", init_globals={
                     'argv': ['--sleep-for-animation=False', '--do_rendering=False']
                 })
 
-            cProfile.run('my_func()', sort='time')
+            cProfile.run('my_func()', sort='time', filename=outfile)
 
 
 for entry in [entry for entry in importlib_resources.contents('examples') if

tox.ini

@@ -71,13 +71,17 @@ commands =
 base_python = python3.10
 deps =
     -r requirements-dev.txt
+set_env =
+    BENCHMARKS_OUTPUT_FOLDER = {work_dir}/{env_name}/log
 commands =
     python benchmarks/benchmark_all_examples.py
 
 [testenv:profiling]
 base_python = python3.10
 deps =
     -r requirements-dev.txt
+set_env =
+    PROFILING_OUTPUT_FOLDER = {work_dir}/{env_name}/log
 commands =
     python benchmarks/profile_all_examples.py