Skip to content

Commit 9501ef8

Browse files
xlqianxlqian
authored
Merge pull request #4225 from hove-io/add_excluded_zones_manager
[Jormungandr] Add excluded zones manager
2 parents 49d02cd + c5caa78 commit 9501ef8

File tree

5 files changed

+139
-30
lines changed

5 files changed

+139
-30
lines changed

‎source/jormungandr/jormungandr/default_settings.py

+2
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
'FETCH_S3_DATA_TIMEOUT': 24 * 60,
158158
# TIMEOUT_TRANSFER_PATH = 24Hours
159159
"TIMEOUT_TRANSFER_PATH": 24 * 60 * 60,
160+
'ASGARD_S3_DATA_TIMEOUT': 2 * 60,
160161
}
161162

162163

@@ -168,6 +169,7 @@
168169
'TIMEOUT_AUTHENTICATION': 30,
169170
'TIMEOUT_PARAMS': 30,
170171
"FETCH_S3_DATA_TIMEOUT": 2 * 60,
172+
'ASGARD_S3_DATA_TIMEOUT': 1 * 60,
171173
}
172174

173175
MEMORY_CACHE_CONFIGURATION = (

‎source/jormungandr/jormungandr/excluded_zones_manager.py

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# Copyright (c) 2001-2024, Hove and/or its affiliates. All rights reserved.
2+
#
3+
# This file is part of Navitia,
4+
# the software to build cool stuff with public transport.
5+
#
6+
# Hope you'll enjoy and contribute to this project,
7+
# powered by Hove (www.hove.com).
8+
# Help us simplify mobility and open public transport:
9+
# a non ending quest to the responsive locomotion way of traveling!
10+
#
11+
# LICENCE: This program is free software; you can redistribute it and/or modify
12+
# it under the terms of the GNU Affero General Public License as published by
13+
# the Free Software Foundation, either version 3 of the License, or
14+
# (at your option) any later version.
15+
#
16+
# This program is distributed in the hope that it will be useful,
17+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
# GNU Affero General Public License for more details.
20+
#
21+
# You should have received a copy of the GNU Affero General Public License
22+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
23+
#
24+
# Stay tuned using
25+
# twitter @navitia
26+
# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org
27+
# https://groups.google.com/d/forum/navitia
28+
# www.navitia.io
29+
30+
import boto3
31+
from botocore.client import Config
32+
import logging
33+
import json
34+
35+
from jormungandr import app, memory_cache, cache
36+
from jormungandr.resource_s3_object import ResourceS3Object
37+
38+
39+
class ExcludedZonesManager:
40+
@staticmethod
41+
@cache.memoize(app.config[str('CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 24 * 60))
42+
def get_object(resource_s3_object):
43+
logger = logging.getLogger(__name__)
44+
try:
45+
file_content = resource_s3_object.s3_object.get()['Body'].read().decode('utf-8')
46+
return json.loads(file_content)
47+
except Exception:
48+
logger.exception('Error while loading file: {}'.format(resource_s3_object.s3_object.key))
49+
return {}
50+
51+
@staticmethod
52+
@memory_cache.memoize(
53+
app.config[str('MEMORY_CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 5 * 60)
54+
)
55+
def get_excluded_zones(instance_name=None, mode=None):
56+
bucket_name = app.config.get(str("ASGARD_S3_BUCKET"))
57+
folder = "excluded_zones"
58+
59+
logger = logging.getLogger(__name__)
60+
args = {"connect_timeout": 2, "read_timeout": 2, "retries": {'max_attempts': 0}}
61+
s3_resource = boto3.resource('s3', config=Config(**args))
62+
excluded_zones = []
63+
try:
64+
my_bucket = s3_resource.Bucket(bucket_name)
65+
for obj in my_bucket.objects.filter(Prefix="{}/".format(folder)):
66+
if not obj.key.endswith('.json'):
67+
continue
68+
try:
69+
json_content = ExcludedZonesManager.get_object(ResourceS3Object(obj, None))
70+
71+
if instance_name is not None and json_content.get('instance') != instance_name:
72+
continue
73+
if mode is not None and mode not in json_content.get("modes", []):
74+
continue
75+
76+
excluded_zones.append(json_content)
77+
except Exception:
78+
logger.exception(
79+
"Error on fetching excluded zones: bucket: {}, instance: {}, mode ={}",
80+
bucket_name,
81+
instance_name,
82+
mode,
83+
)
84+
continue
85+
86+
except Exception:
87+
logger.exception(
88+
"Error on fetching excluded zones: bucket: {}, instance: {}, mode ={}",
89+
bucket_name,
90+
instance_name,
91+
mode,
92+
)
93+
94+
return excluded_zones

‎source/jormungandr/jormungandr/interfaces/v1/opg_excluded_zones.py

+4-20
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from jormungandr.interfaces.v1.StatedResource import StatedResource
3636
from jormungandr.interfaces.v1.make_links import create_external_link
3737
from jormungandr import i_manager, app
38+
from jormungandr.excluded_zones_manager import ExcludedZonesManager
3839

3940

4041
class OpgExcludedZones(StatedResource):
@@ -74,27 +75,10 @@ def fetch_and_get_data(self, instance, bucket_name, folder, mode=None):
7475
return {}
7576
places = []
7677
logger = logging.getLogger(__name__)
77-
args = {"connect_timeout": 2, "read_timeout": 2, "retries": {'max_attempts': 0}}
78-
s3_resource = boto3.resource('s3', config=Config(**args))
7978
try:
80-
my_bucket = s3_resource.Bucket(bucket_name)
81-
for obj in my_bucket.objects.filter(Prefix="{}/".format(folder)):
82-
if not obj.key.endswith('.json'):
83-
continue
84-
try:
85-
file_content = obj.get()['Body'].read().decode('utf-8')
86-
json_content = json.loads(file_content)
87-
if json_content.get('instance') != instance.name:
88-
continue
89-
if mode is not None:
90-
if mode not in json_content.get("modes"):
91-
continue
92-
place = self.get_poi_place(instance, json_content)
93-
places.append(place)
94-
95-
except Exception:
96-
logger.exception("Error on OpgExcludedZones")
97-
continue
79+
for json_content in ExcludedZonesManager.get_excluded_zones(instance.name, mode):
80+
place = self.get_poi_place(instance, json_content)
81+
places.append(place)
9882
except Exception:
9983
logger.exception("Error on OpgExcludedZones")
10084
return {}

‎source/jormungandr/jormungandr/olympic_site_params_manager.py

+2-10
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import logging
3131
import json
3232
from collections import namedtuple
33+
34+
from jormungandr.resource_s3_object import ResourceS3Object
3335
from jormungandr.utils import (
3436
is_olympic_poi,
3537
local_str_date_to_utc,
@@ -38,7 +40,6 @@
3840
str_datetime_utc_to_local,
3941
)
4042
import boto3
41-
from jormungandr import app
4243
from navitiacommon import type_pb2
4344
from botocore.client import Config
4445
from jormungandr import app, memory_cache, cache
@@ -47,15 +48,6 @@
4748
AttractivityVirtualFallback = namedtuple("AttractivityVirtualFallback", "attractivity, virtual_duration")
4849

4950

50-
class ResourceS3Object:
51-
def __init__(self, s3_object, instance_name):
52-
self.s3_object = s3_object
53-
self.instance_name = instance_name
54-
55-
def __repr__(self):
56-
return "{}-{}-{}".format(self.instance_name, self.s3_object.key, self.s3_object.e_tag)
57-
58-
5951
def has_applicable_scenario(api_request):
6052
return bool(api_request.get("olympic_site_params"))
6153

‎source/jormungandr/jormungandr/resource_s3_object.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright (c) 2001-2024, Hove and/or its affiliates. All rights reserved.
2+
#
3+
# This file is part of Navitia,
4+
# the software to build cool stuff with public transport.
5+
#
6+
# Hope you'll enjoy and contribute to this project,
7+
# powered by Hove (www.hove.com).
8+
# Help us simplify mobility and open public transport:
9+
# a non ending quest to the responsive locomotion way of traveling!
10+
#
11+
# LICENCE: This program is free software; you can redistribute it and/or modify
12+
# it under the terms of the GNU Affero General Public License as published by
13+
# the Free Software Foundation, either version 3 of the License, or
14+
# (at your option) any later version.
15+
#
16+
# This program is distributed in the hope that it will be useful,
17+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
# GNU Affero General Public License for more details.
20+
#
21+
# You should have received a copy of the GNU Affero General Public License
22+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
23+
#
24+
# Stay tuned using
25+
# twitter @navitia
26+
# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org
27+
# https://groups.google.com/d/forum/navitia
28+
# www.navitia.io
29+
30+
31+
class ResourceS3Object:
32+
def __init__(self, s3_object, instance_name):
33+
self.s3_object = s3_object
34+
self.instance_name = instance_name
35+
36+
def __repr__(self):
37+
return "{}-{}-{}".format(self.instance_name, self.s3_object.key, self.s3_object.e_tag)

0 commit comments

Comments
 (0)