Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make update script less verbose by default #2033

Merged
1 commit merged into from
Aug 11, 2021
Merged

Make update script less verbose by default #2033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions intelmq/bots/experts/asn_lookup/expert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def run(cls, parsed_args=None):
parsed_args = cls._create_argparser().parse_args()

if parsed_args.update_database:
cls.update_database()
cls.update_database(verbose=parsed_args.verbose)

else:
super().run(parsed_args=parsed_args)
Expand All @@ -87,10 +87,11 @@ def run(cls, parsed_args=None):
def _create_argparser(cls):
argparser = super()._create_argparser()
argparser.add_argument("--update-database", action='store_true', help='downloads latest database data')
argparser.add_argument("--verbose", action='store_true', help='be verbose')
return argparser

@classmethod
def update_database(cls):
def update_database(cls, verbose=False):
bots = {}
runtime_conf = get_bots_settings()
try:
Expand All @@ -102,15 +103,17 @@ def update_database(cls):
sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e))

if not bots:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
if verbose:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
sys.exit(0)

# we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed
if pyasn is None:
raise MissingDependencyError("pyasn")

try:
print("Searching for the latest database update...")
if verbose:
print("Searching for the latest database update...")
session = create_request_session()
url = "http://archive.routeviews.org/route-views4/bgpdata/"
response = session.get(url)
Expand All @@ -130,7 +133,8 @@ def update_database(cls):
if not days:
sys.exit("Database update failed. Couldn't find the latest database update.")

print("Downloading the latest database update...")
if verbose:
print("Downloading the latest database update...")
url += days[0]
response = session.get(url)

Expand All @@ -142,15 +146,17 @@ def update_database(cls):
sys.exit("Database update failed. Connection Error: {0}".format(e))

with bz2.open(io.BytesIO(response.content)) as archive:
print("Parsing the latest database update...")
if verbose:
print("Parsing the latest database update...")
prefixes = pyasn.mrtx.parse_mrt_file(archive, print_progress=False, skip_record_on_error=True)

for database_path in set(bots.values()):
database_dir = pathlib.Path(database_path).parent
database_dir.mkdir(parents=True, exist_ok=True)
pyasn.mrtx.dump_prefixes_to_file(prefixes, database_path)

print("Database updated. Reloading affected bots.")
if verbose:
print("Database updated. Reloading affected bots.")

ctl = IntelMQController()
for bot in bots.keys():
Expand Down
11 changes: 7 additions & 4 deletions intelmq/bots/experts/domain_suffix/expert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def run(cls, parsed_args=None):
parsed_args = cls._create_argparser().parse_args()

if parsed_args.update_database:
cls.update_database()
cls.update_database(verbose=parsed_args.verbose)

else:
super().run(parsed_args=parsed_args)
Expand All @@ -75,10 +75,11 @@ def run(cls, parsed_args=None):
def _create_argparser(cls):
argparser = super()._create_argparser()
argparser.add_argument("--update-database", action='store_true', help='downloads latest database data')
argparser.add_argument("--verbose", action='store_true', help='be verbose')
return argparser

@classmethod
def update_database(cls):
def update_database(cls, verbose=False):
bots = {}
runtime_conf = get_bots_settings()
try:
Expand All @@ -98,7 +99,8 @@ def update_database(cls):
try:
session = create_request_session()
url = "https://publicsuffix.org/list/public_suffix_list.dat"
print("Downloading the latest database update...")
if verbose:
print("Downloading the latest database update...")
response = session.get(url)

if not response.ok:
Expand All @@ -114,7 +116,8 @@ def update_database(cls):
with open(database_path, "wb") as database:
database.write(response.content)

print("Database updated. Reloading affected bots.")
if verbose:
print("Database updated. Reloading affected bots.")

ctl = IntelMQController()
for bot in bots.keys():
Expand Down
14 changes: 9 additions & 5 deletions intelmq/bots/experts/maxmind_geoip/expert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def run(cls, parsed_args=None):
parsed_args = cls._create_argparser().parse_args()

if parsed_args.update_database:
cls.update_database()
cls.update_database(verbose=parsed_args.verbose)

else:
super().run(parsed_args=parsed_args)
Expand All @@ -103,10 +103,11 @@ def run(cls, parsed_args=None):
def _create_argparser(cls):
argparser = super()._create_argparser()
argparser.add_argument("--update-database", action='store_true', help='downloads latest database data')
argparser.add_argument("--verbose", action='store_true', help='be verbose')
return argparser

@classmethod
def update_database(cls):
def update_database(cls, verbose=False):
bots = {}
license_key = None
runtime_conf = get_bots_settings()
Expand All @@ -127,7 +128,8 @@ def update_database(cls):
sys.exit(error)

if not bots:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
if verbose:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
sys.exit(0)

# we only need to import now, if there are no maxmind_geoip bots, this dependency does not need to be installed
Expand All @@ -139,7 +141,8 @@ def update_database(cls):
"is a dependency for the required geoip2 package.")

try:
print("Downloading the latest database update...")
if verbose:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get("https://download.maxmind.com/app/geoip_download",
params={
Expand Down Expand Up @@ -179,7 +182,8 @@ def update_database(cls):
with open(database_path, "wb") as database:
database.write(database_data._buffer)

print("Database updated. Reloading affected bots.")
if verbose:
print("Database updated. Reloading affected bots.")

ctl = IntelMQController()
for bot in bots.keys():
Expand Down
14 changes: 9 additions & 5 deletions intelmq/bots/experts/recordedfuture_iprisk/expert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def run(cls, parsed_args=None):
parsed_args = cls._create_argparser().parse_args()

if parsed_args.update_database:
cls.update_database()
cls.update_database(verbose=parsed_args.verbose)

else:
super().run(parsed_args=parsed_args)
Expand All @@ -67,10 +67,11 @@ def run(cls, parsed_args=None):
def _create_argparser(cls):
argparser = super()._create_argparser()
argparser.add_argument("--update-database", action='store_true', help='downloads latest database data')
argparser.add_argument("--verbose", action='store_true', help='be verbose')
return argparser

@classmethod
def update_database(cls):
def update_database(cls, verbose=False):
bots = {}
api_token = None
runtime_conf = get_bots_settings()
Expand All @@ -84,11 +85,13 @@ def update_database(cls):
sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e))

if not bots:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
if verbose:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
sys.exit(0)

try:
print("Downloading the latest database update...")
if verbose:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get("https://api.recordedfuture.com/v2/ip/risklist",
params={
Expand Down Expand Up @@ -127,7 +130,8 @@ def update_database(cls):
with open(database_path, "w") as database:
database.write(database_data)

print("Database updated. Reloading affected bots.")
if verbose:
print("Database updated. Reloading affected bots.")

ctl = IntelMQController()
for bot in bots.keys():
Expand Down
14 changes: 9 additions & 5 deletions intelmq/bots/experts/tor_nodes/expert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def run(cls, parsed_args=None):
parsed_args = cls._create_argparser().parse_args()

if parsed_args.update_database:
cls.update_database()
cls.update_database(verbose=parsed_args.verbose)

else:
super().run(parsed_args=parsed_args)
Expand All @@ -69,10 +69,11 @@ def run(cls, parsed_args=None):
def _create_argparser(cls):
argparser = super()._create_argparser()
argparser.add_argument("--update-database", action='store_true', help='downloads latest database data')
argparser.add_argument("--verbose", action='store_true', help='be verbose')
return argparser

@classmethod
def update_database(cls):
def update_database(cls, verbose=False):
bots = {}
runtime_conf = get_bots_settings()
try:
Expand All @@ -84,11 +85,13 @@ def update_database(cls):
sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e))

if not bots:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
if verbose:
print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__))
sys.exit(0)

try:
print("Downloading the latest database update...")
if verbose:
print("Downloading the latest database update...")
session = create_request_session()
response = session.get("https://check.torproject.org/exit-addresses")
except requests.exceptions.RequestException as e:
Expand All @@ -107,7 +110,8 @@ def update_database(cls):
with open(database_path, "w") as database:
database.write(tor_exits)

print("Database updated. Reloading affected bots.")
if verbose:
print("Database updated. Reloading affected bots.")

ctl = IntelMQController()
for bot in bots.keys():
Expand Down