BUG: bots: ignore NULL bytes in all csv parsers

fixes #967
certtools · Nov 9, 2017 · 62e7a94 · 62e7a94
1 parent b62b151
commit 62e7a94
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ CHANGELOG
 
 ### Bots
 #### Parsers
+- All CSV parsers ignore NULL-bytes now, because the csv-library cannot handle it (#967)
 - Modify Bot default ruleset: changed conficker rule to catch more spellings
 
 ### Documentation

diff --git a/intelmq/bots/parsers/abusech/parser_ransomware.py b/intelmq/bots/parsers/abusech/parser_ransomware.py
@@ -23,6 +23,7 @@ def process(self):
 
         report = self.receive_message()
         raw_report = utils.base64_decode(report.get("raw"))
+        raw_report = raw_report.translate({0: None})
 
         for row in csv.reader(io.StringIO(raw_report)):
             if row[0].startswith('#'):

diff --git a/intelmq/bots/parsers/generic/parser_csv.py b/intelmq/bots/parsers/generic/parser_csv.py
@@ -50,6 +50,7 @@ def init(self):
 
     def parse(self, report):
         raw_report = utils.base64_decode(report.get("raw"))
+        raw_report = raw_report.translate({0: None})
         # ignore lines starting with #
         raw_report = re.sub(r'(?m)^#.*\n?', '', raw_report)
         # ignore null bytes

diff --git a/intelmq/bots/parsers/phishtank/parser.py b/intelmq/bots/parsers/phishtank/parser.py
@@ -22,6 +22,7 @@ def process(self):
                    ]
 
         raw_report = utils.base64_decode(report.get("raw"))
+        raw_report = raw_report.translate({0: None})
         for row in csv.reader(io.StringIO(raw_report)):
 
             if not len(row):  # csv module can give empty lists

diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py
@@ -49,7 +49,6 @@ def init(self):
 
     def parse(self, report):
         raw_report = utils.base64_decode(report["raw"])
-        # Temporary fix for https://github.com/certtools/intelmq/issues/967
         raw_report = raw_report.translate({0: None})
         csvr = csv.DictReader(io.StringIO(raw_report))
 

diff --git a/intelmq/bots/parsers/turris/parser.py b/intelmq/bots/parsers/turris/parser.py
@@ -20,6 +20,7 @@ def process(self):
 
         headers = True
         raw_report = utils.base64_decode(report.get("raw"))
+        raw_report = raw_report.translate({0: None})
         for row in csv.reader(io.StringIO(raw_report)):
             # ignore headers
             if headers:

diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py
@@ -545,6 +545,7 @@ def parse_csv(self, report: dict):
         A basic CSV parser.
         """
         raw_report = utils.base64_decode(report.get("raw")).strip()
+        raw_report = raw_report.translate({0: None})
         if self.ignore_lines_starting:
             raw_report = '\n'.join([line for line in raw_report.splitlines()
                                     if not any([line.startswith(prefix) for prefix
@@ -558,6 +559,7 @@ def parse_csv_dict(self, report: dict):
         A basic CSV Dictionary parser.
         """
         raw_report = utils.base64_decode(report.get("raw")).strip()
+        raw_report = raw_report.translate({0: None})
         if self.ignore_lines_starting:
             raw_report = '\n'.join([line for line in raw_report.splitlines()
                                     if not any([line.startswith(prefix) for prefix