diff --git a/intelmq/bots/experts/asn_lookup/expert.py b/intelmq/bots/experts/asn_lookup/expert.py index 257f75660..b75ab5c93 100644 --- a/intelmq/bots/experts/asn_lookup/expert.py +++ b/intelmq/bots/experts/asn_lookup/expert.py @@ -78,7 +78,7 @@ def run(cls, parsed_args=None): parsed_args = cls._create_argparser().parse_args() if parsed_args.update_database: - cls.update_database() + cls.update_database(verbose=parsed_args.verbose) else: super().run(parsed_args=parsed_args) @@ -87,10 +87,11 @@ def run(cls, parsed_args=None): def _create_argparser(cls): argparser = super()._create_argparser() argparser.add_argument("--update-database", action='store_true', help='downloads latest database data') + argparser.add_argument("--verbose", action='store_true', help='be verbose') return argparser @classmethod - def update_database(cls): + def update_database(cls, verbose=False): bots = {} runtime_conf = get_bots_settings() try: @@ -102,7 +103,8 @@ def update_database(cls): sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e)) if not bots: - print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) + if verbose: + print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) sys.exit(0) # we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed @@ -110,7 +112,8 @@ def update_database(cls): raise MissingDependencyError("pyasn") try: - print("Searching for the latest database update...") + if verbose: + print("Searching for the latest database update...") session = create_request_session() url = "http://archive.routeviews.org/route-views4/bgpdata/" response = session.get(url) @@ -130,7 +133,8 @@ def update_database(cls): if not days: sys.exit("Database update failed. Couldn't find the latest database update.") - print("Downloading the latest database update...") + if verbose: + print("Downloading the latest database update...") url += days[0] response = session.get(url) @@ -142,7 +146,8 @@ def update_database(cls): sys.exit("Database update failed. Connection Error: {0}".format(e)) with bz2.open(io.BytesIO(response.content)) as archive: - print("Parsing the latest database update...") + if verbose: + print("Parsing the latest database update...") prefixes = pyasn.mrtx.parse_mrt_file(archive, print_progress=False, skip_record_on_error=True) for database_path in set(bots.values()): @@ -150,7 +155,8 @@ def update_database(cls): database_dir.mkdir(parents=True, exist_ok=True) pyasn.mrtx.dump_prefixes_to_file(prefixes, database_path) - print("Database updated. Reloading affected bots.") + if verbose: + print("Database updated. Reloading affected bots.") ctl = IntelMQController() for bot in bots.keys(): diff --git a/intelmq/bots/experts/domain_suffix/expert.py b/intelmq/bots/experts/domain_suffix/expert.py index 265fc6f25..eb2fc043c 100644 --- a/intelmq/bots/experts/domain_suffix/expert.py +++ b/intelmq/bots/experts/domain_suffix/expert.py @@ -66,7 +66,7 @@ def run(cls, parsed_args=None): parsed_args = cls._create_argparser().parse_args() if parsed_args.update_database: - cls.update_database() + cls.update_database(verbose=parsed_args.verbose) else: super().run(parsed_args=parsed_args) @@ -75,10 +75,11 @@ def run(cls, parsed_args=None): def _create_argparser(cls): argparser = super()._create_argparser() argparser.add_argument("--update-database", action='store_true', help='downloads latest database data') + argparser.add_argument("--verbose", action='store_true', help='be verbose') return argparser @classmethod - def update_database(cls): + def update_database(cls, verbose=False): bots = {} runtime_conf = get_bots_settings() try: @@ -98,7 +99,8 @@ def update_database(cls): try: session = create_request_session() url = "https://publicsuffix.org/list/public_suffix_list.dat" - print("Downloading the latest database update...") + if verbose: + print("Downloading the latest database update...") response = session.get(url) if not response.ok: @@ -114,7 +116,8 @@ def update_database(cls): with open(database_path, "wb") as database: database.write(response.content) - print("Database updated. Reloading affected bots.") + if verbose: + print("Database updated. Reloading affected bots.") ctl = IntelMQController() for bot in bots.keys(): diff --git a/intelmq/bots/experts/maxmind_geoip/expert.py b/intelmq/bots/experts/maxmind_geoip/expert.py index 6ca1150c0..5e06f56db 100644 --- a/intelmq/bots/experts/maxmind_geoip/expert.py +++ b/intelmq/bots/experts/maxmind_geoip/expert.py @@ -94,7 +94,7 @@ def run(cls, parsed_args=None): parsed_args = cls._create_argparser().parse_args() if parsed_args.update_database: - cls.update_database() + cls.update_database(verbose=parsed_args.verbose) else: super().run(parsed_args=parsed_args) @@ -103,10 +103,11 @@ def run(cls, parsed_args=None): def _create_argparser(cls): argparser = super()._create_argparser() argparser.add_argument("--update-database", action='store_true', help='downloads latest database data') + argparser.add_argument("--verbose", action='store_true', help='be verbose') return argparser @classmethod - def update_database(cls): + def update_database(cls, verbose=False): bots = {} license_key = None runtime_conf = get_bots_settings() @@ -127,7 +128,8 @@ def update_database(cls): sys.exit(error) if not bots: - print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) + if verbose: + print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) sys.exit(0) # we only need to import now, if there are no maxmind_geoip bots, this dependency does not need to be installed @@ -139,7 +141,8 @@ def update_database(cls): "is a dependency for the required geoip2 package.") try: - print("Downloading the latest database update...") + if verbose: + print("Downloading the latest database update...") session = create_request_session() response = session.get("https://download.maxmind.com/app/geoip_download", params={ @@ -179,7 +182,8 @@ def update_database(cls): with open(database_path, "wb") as database: database.write(database_data._buffer) - print("Database updated. Reloading affected bots.") + if verbose: + print("Database updated. Reloading affected bots.") ctl = IntelMQController() for bot in bots.keys(): diff --git a/intelmq/bots/experts/recordedfuture_iprisk/expert.py b/intelmq/bots/experts/recordedfuture_iprisk/expert.py index 88ff95d71..cb66011d8 100644 --- a/intelmq/bots/experts/recordedfuture_iprisk/expert.py +++ b/intelmq/bots/experts/recordedfuture_iprisk/expert.py @@ -58,7 +58,7 @@ def run(cls, parsed_args=None): parsed_args = cls._create_argparser().parse_args() if parsed_args.update_database: - cls.update_database() + cls.update_database(verbose=parsed_args.verbose) else: super().run(parsed_args=parsed_args) @@ -67,10 +67,11 @@ def run(cls, parsed_args=None): def _create_argparser(cls): argparser = super()._create_argparser() argparser.add_argument("--update-database", action='store_true', help='downloads latest database data') + argparser.add_argument("--verbose", action='store_true', help='be verbose') return argparser @classmethod - def update_database(cls): + def update_database(cls, verbose=False): bots = {} api_token = None runtime_conf = get_bots_settings() @@ -84,11 +85,13 @@ def update_database(cls): sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e)) if not bots: - print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) + if verbose: + print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) sys.exit(0) try: - print("Downloading the latest database update...") + if verbose: + print("Downloading the latest database update...") session = create_request_session() response = session.get("https://api.recordedfuture.com/v2/ip/risklist", params={ @@ -127,7 +130,8 @@ def update_database(cls): with open(database_path, "w") as database: database.write(database_data) - print("Database updated. Reloading affected bots.") + if verbose: + print("Database updated. Reloading affected bots.") ctl = IntelMQController() for bot in bots.keys(): diff --git a/intelmq/bots/experts/tor_nodes/expert.py b/intelmq/bots/experts/tor_nodes/expert.py index 3aaa11d40..5b9e10e82 100644 --- a/intelmq/bots/experts/tor_nodes/expert.py +++ b/intelmq/bots/experts/tor_nodes/expert.py @@ -60,7 +60,7 @@ def run(cls, parsed_args=None): parsed_args = cls._create_argparser().parse_args() if parsed_args.update_database: - cls.update_database() + cls.update_database(verbose=parsed_args.verbose) else: super().run(parsed_args=parsed_args) @@ -69,10 +69,11 @@ def run(cls, parsed_args=None): def _create_argparser(cls): argparser = super()._create_argparser() argparser.add_argument("--update-database", action='store_true', help='downloads latest database data') + argparser.add_argument("--verbose", action='store_true', help='be verbose') return argparser @classmethod - def update_database(cls): + def update_database(cls, verbose=False): bots = {} runtime_conf = get_bots_settings() try: @@ -84,11 +85,13 @@ def update_database(cls): sys.exit("Database update failed. Your configuration of {0} is missing key {1}.".format(bot, e)) if not bots: - print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) + if verbose: + print("Database update skipped. No bots of type {0} present in runtime.conf.".format(__name__)) sys.exit(0) try: - print("Downloading the latest database update...") + if verbose: + print("Downloading the latest database update...") session = create_request_session() response = session.get("https://check.torproject.org/exit-addresses") except requests.exceptions.RequestException as e: @@ -107,7 +110,8 @@ def update_database(cls): with open(database_path, "w") as database: database.write(tor_exits) - print("Database updated. Reloading affected bots.") + if verbose: + print("Database updated. Reloading affected bots.") ctl = IntelMQController() for bot in bots.keys():