Merge pull request #4264 from hove-io/hot_fix_add_cache_to_ZE_for_stop_area

[Distributed/Excluded Zones] Add cache to ZE for stop area

Author: xlqian

source/jormungandr/jormungandr/default_settings.py

@@ -4,6 +4,7 @@
 import os
 import json
 from flask_restful.inputs import boolean
+from datetime import timedelta
 
 # sql queries will return an exception if the query did not succeed before `statement_timeout`
 DEFAULT_SQLALCHEMY_ENGINE_OPTIONS = {
@@ -157,7 +158,7 @@
     'FETCH_S3_DATA_TIMEOUT': 24 * 60,
     # TIMEOUT_TRANSFER_PATH = 24Hours
     "TIMEOUT_TRANSFER_PATH": 24 * 60 * 60,
-    'ASGARD_S3_DATA_TIMEOUT': 2 * 60,
+    'ASGARD_S3_DATA_TIMEOUT': timedelta(hours=2).total_seconds(),
 }
 
 
@@ -169,7 +170,7 @@
     'TIMEOUT_AUTHENTICATION': 30,
     'TIMEOUT_PARAMS': 30,
     "FETCH_S3_DATA_TIMEOUT": 2 * 60,
-    'ASGARD_S3_DATA_TIMEOUT': 1 * 60,
+    'ASGARD_S3_DATA_TIMEOUT': timedelta(minutes=30).total_seconds(),
 }
 
 MEMORY_CACHE_CONFIGURATION = (

source/jormungandr/jormungandr/excluded_zones_manager.py

@@ -28,27 +28,34 @@
 # www.navitia.io
 
 import boto3
-import pytz
 import shapely.iterops
 from dateutil import parser
 from botocore.client import Config
 import logging
 import json
 import shapely
-import datetime
 from typing import Dict
+from datetime import timedelta
 
 from jormungandr import app, memory_cache, cache
 from jormungandr.resource_s3_object import ResourceS3Object
 
+# local memory cache
+MEMORY_CACHE_ASGARD_S3_DATA_TIMEOUT = app.config[str('MEMORY_CACHE_CONFIGURATION')].get(
+    str('ASGARD_S3_DATA_TIMEOUT'), timedelta(minutes=30).total_seconds()
+)
+# Redis cache
+CACHE_ASGARD_S3_DATA_TIMEOUT = app.config[str('CACHE_CONFIGURATION')].get(
+    str('ASGARD_S3_DATA_TIMEOUT'), timedelta(hours=2).total_seconds()
+)
+
 
 class ExcludedZonesManager:
     excluded_shapes = dict()  # type: Dict[str, shapely.geometry]
     asgard_s3_bucket_folder = "excluded_zones"
 
     @staticmethod
-    @cache.memoize(app.config[str('CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 24 * 60))
-    def get_object(resource_s3_object):
+    def _get_object(resource_s3_object):
         logger = logging.getLogger(__name__)
         try:
             file_content = resource_s3_object.s3_object.get()['Body'].read().decode('utf-8')
@@ -58,7 +65,7 @@ def get_object(resource_s3_object):
             return {}
 
     @staticmethod
-    def is_activated(activation_period, date):
+    def _is_activated(activation_period, date):
         if activation_period is None:
             return False
 
@@ -70,10 +77,9 @@ def is_between(period, d):
         return any((is_between(period, date) for period in activation_period))
 
     @classmethod
-    @memory_cache.memoize(
-        app.config[str('MEMORY_CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 10 * 60)
-    )
-    def get_all_excluded_zones(cls):
+    @memory_cache.memoize(MEMORY_CACHE_ASGARD_S3_DATA_TIMEOUT)
+    @cache.memoize(CACHE_ASGARD_S3_DATA_TIMEOUT)
+    def _get_all_excluded_zones(cls):
         bucket_name = app.config.get(str("ASGARD_S3_BUCKET"))
 
         logger = logging.getLogger(__name__)
@@ -86,7 +92,7 @@ def get_all_excluded_zones(cls):
                 if not obj.key.endswith('.json'):
                     continue
                 try:
-                    json_content = ExcludedZonesManager.get_object(ResourceS3Object(obj, None))
+                    json_content = ExcludedZonesManager._get_object(ResourceS3Object(obj, None))
                     excluded_zones.append(json_content)
                 except Exception:
                     logger.exception("Error on fetching excluded zones: bucket: {}", bucket_name)
@@ -114,25 +120,24 @@ def get_all_excluded_zones(cls):
         return excluded_zones
 
     @staticmethod
-    @memory_cache.memoize(
-        app.config[str('MEMORY_CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 10 * 60)
-    )
+    @memory_cache.memoize(MEMORY_CACHE_ASGARD_S3_DATA_TIMEOUT)
+    @cache.memoize(CACHE_ASGARD_S3_DATA_TIMEOUT)
     def get_excluded_zones(mode=None, date=None):
         excluded_zones = []
-        for json_content in ExcludedZonesManager.get_all_excluded_zones():
+        for json_content in ExcludedZonesManager._get_all_excluded_zones():
             if mode is not None and mode not in json_content.get("modes", []):
                 continue
-            if date is not None and not ExcludedZonesManager.is_activated(
+            if date is not None and not ExcludedZonesManager._is_activated(
                 json_content.get('activation_periods'), date
             ):
                 continue
             excluded_zones.append(json_content)
         return excluded_zones
 
     @classmethod
-    @memory_cache.memoize(app.config[str('CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 10 * 60))
-    def is_excluded(cls, obj, mode, timestamp):
-        date = datetime.datetime.fromtimestamp(timestamp, tz=pytz.timezone("UTC")).date()
+    @memory_cache.memoize(MEMORY_CACHE_ASGARD_S3_DATA_TIMEOUT)
+    @cache.memoize(CACHE_ASGARD_S3_DATA_TIMEOUT)
+    def is_excluded(cls, obj, mode, date):
         # update excluded zones
         excluded_zones = ExcludedZonesManager.get_excluded_zones(mode=mode, date=date)
         poi_ids = set((zone.get("poi") for zone in excluded_zones))

source/jormungandr/jormungandr/scenarios/helper_classes/fallback_durations.py

@@ -47,6 +47,8 @@
 from jormungandr.scenarios.helper_classes.places_free_access import FreeAccessObject
 import functools
 import itertools
+import pytz
+import datetime
 
 # The basic element stored in fallback_durations.
 # in DurationElement. can be found:
@@ -202,10 +204,12 @@ def _filter_free_access_with_excluded_zones(self, all_free_access):
         if self._request['_use_excluded_zones'] and all_free_access:
             # the mode is hardcoded to walking because we consider that we access to all free_access places
             # by walking
+            timestamp = self._request['datetime']
+            date = datetime.datetime.fromtimestamp(timestamp, tz=pytz.timezone("UTC")).date()
             is_excluded = functools.partial(
                 excluded_zones_manager.ExcludedZonesManager.is_excluded,
                 mode='walking',
-                timestamp=self._request['datetime'],
+                date=date,
             )
             return set(itertools.filterfalse(is_excluded, all_free_access))
         return all_free_access

source/jormungandr/jormungandr/tests/excluded_zones_manager_tests.py

@@ -64,7 +64,7 @@ def mock_get_all_excluded_zones():
 @pytest.fixture(scope="function", autouse=True)
 def mock_http_karos(monkeypatch):
     monkeypatch.setattr(
-        'jormungandr.excluded_zones_manager.ExcludedZonesManager.get_all_excluded_zones',
+        'jormungandr.excluded_zones_manager.ExcludedZonesManager._get_all_excluded_zones',
         mock_get_all_excluded_zones,
     )
 
@@ -92,15 +92,25 @@ def get_excluded_zones_test():
 
 
 def excluded_manager_is_excluded_test():
+    import datetime
+    import pytz
+
     Coord = namedtuple('Coord', ['lon', 'lat'])
 
     ExcludedZonesManager.excluded_shapes["poi:gare_de_lyon"] = shapely.wkt.loads(shape_str)
 
-    res = ExcludedZonesManager.is_excluded(Coord(1, 1), mode='walking', timestamp=1723280205)
+    def ts_to_date(timestamp):
+        return datetime.datetime.fromtimestamp(timestamp, tz=pytz.timezone("UTC")).date()
+
+    res = ExcludedZonesManager.is_excluded(Coord(1, 1), mode='walking', date=ts_to_date(1723280205))
     assert not res
 
-    res = ExcludedZonesManager.is_excluded(Coord(2.376365, 48.843402), mode='walking', timestamp=1723280205)
+    res = ExcludedZonesManager.is_excluded(
+        Coord(2.376365, 48.843402), mode='walking', date=ts_to_date(1723280205)
+    )
     assert res
 
-    res = ExcludedZonesManager.is_excluded(Coord(2.376365, 48.843402), mode='walking', timestamp=1722502605)
+    res = ExcludedZonesManager.is_excluded(
+        Coord(2.376365, 48.843402), mode='walking', date=ts_to_date(1722502605)
+    )
     assert not res