Merge branch 'maintenance'

.travis.yml

@@ -11,7 +11,7 @@ python:
   - "3.5"
   - "3.6"
 install:
-  - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot
+  - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot dh-systemd
   - if [[ $REQUIREMENTS == true ]] ; then for file in intelmq/bots/*/*/REQUIREMENTS.txt; do pip install -r $file; done; fi
   - if [[ $TRAVIS_PYTHON_VERSION < '3.5' ]]; then pip install typing; fi
   - pip install codecov
@@ -26,10 +26,19 @@ before_script:
   - intelmq_psql_initdb
   - sed -i 's/events/tests/g' /tmp/initdb.sql
   - psql -v ON_ERROR_STOP=on -f /tmp/initdb.sql intelmq -U intelmq
+  - VERSION=$(git describe --abbrev=0 --tags)
+  - git archive --format=tar.gz HEAD > ../intelmq_$VERSION.orig.tar.gz
+  - git archive --format=tar.gz --prefix=debian/ HEAD:debian/ > ../intelmq_$VERSION-1.debian.tar.gz
+  - pushd ..
+  - mkdir build
+  - cd build
+  - tar -xzf ../intelmq_$VERSION.orig.tar.gz
+  - tar -xzf ../intelmq_$VERSION-1.debian.tar.gz
+  - popd
 script:
   - if [[ $REQUIREMENTS == true ]] ; then INTELMQ_TEST_DATABASES=1 INTELMQ_TEST_LOCAL_WEB=1 INTELMQ_TEST_EXOTIC=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; else INTELMQ_TEST_LOCAL_WEB=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; fi
-  - dpkg-buildpackage -us -uc
   - pycodestyle intelmq/{bots,lib,bin}
+  - pushd ../build && dpkg-buildpackage -us -uc; popd
 services:
   - redis-server
   - postgresql

CHANGELOG.md

@@ -1,6 +1,27 @@
 CHANGELOG
 ==========
 
+1.0.2 Bugfix release
+--------------------
+
+### Core
+- `lib.message.add`: parameter force has finally been removed, should have been gone in 1.0.0.rc1 already
+
+### Bots
+- `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852).
+- `parsers.spamhaus.parser_cert`: Handle/ignore 'AS?' in feed (#1111)
+
+### Packaging
+- The following changes have been in effect for the built packages already since version 1.0.0
+- Support building for more distributions, now supported: CentOS 7, Debian 8 and 9, Fedora 25 and 26, RHEL 7, openSUSE Leap 42.2 and 42.3 and Tumbleweed, Ubuntu 14.04 and 16.04
+- Use LSB-paths for created packages (/etc/intelmq/, /var/lib/intelmq/, /run/intelmq/) (#470). Does does not affect installations with setuptools/pip.
+- Change the debian package format from native to quilt
+- Fix problems in postint and postrm scripts
+- Use systemd-tmpfile for creation of /run/intelmq/
+
+### Documentation
+- Add disclaimer on maxmind database in bot documentation and code and the cron-job (#1110)
+
 1.0.1 Bugfix release
 --------------------
 ### Documentation
@@ -66,7 +87,7 @@ CHANGELOG
 - `bots.parsers.alienvault.parser_otx`: handle timestamps without floating point seconds
 
 ### Experts
-- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: true
+- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: False
 
 v1.0.0.dev8
 -----------

NEWS.md

@@ -3,6 +3,10 @@ NEWS
 
 See the changelog for a full list of changes.
 
+1.0.2 Bugfix release
+--------------------
+No changes needed.
+
 1.0.1 Bugfix release
 --------------------
 No changes needed.

contrib/cron-jobs/maxmind

@@ -17,6 +17,9 @@
 # 02  01 *   *   1     ( cd /tmp; /my/path/to/this/script )
 # 
 
+# This product includes GeoLite2 data created by MaxMind, available from
+# <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+
 #export http_proxy=http://proxy.cert.at:8080/
 
 path=/usr/local/bin

debian/changelog

@@ -1,3 +1,15 @@
+intelmq (1.0.2-1) experimental; urgency=low
+
+  * Update to version 1.0.2
+
+ -- Wagner Sebastian <[email protected]>  Mon, 06 Nov 2017 11:36:58 +0100
+
+intelmq (1.0.1-2) experimental; urgency=low
+
+  * New base branch 'packaging' for packages
+
+ -- Wagner Sebastian <[email protected]>  Wed, 20 Sep 2017 16:26:59 +0200
+
 intelmq (1.0.1-1) experimental; urgency=low
 
   * update to version 1.0.1
@@ -20,6 +32,10 @@ intelmq (1.0.0.rc1-1) experimental; urgency=medium
 
   * update to version 1.0.0.rc1
 
+intelmq (1.0.0.dev8-2) experimental; urgency=medium
+
+  * packaging fixes
+
  -- Sebastian Wagner <[email protected]>  Wed, 20 Jun 2017 16:05:00 +0200
 
 intelmq (1.0.0.dev8-1) experimental; urgency=medium

debian/conffiles

@@ -1,5 +1,5 @@
-/opt/intelmq/etc/harmonization.conf
-/opt/intelmq/etc/BOTS
-/opt/intelmq/etc/pipeline.conf
-/opt/intelmq/etc/defaults.conf
-/opt/intelmq/etc/runtime.conf
+/etc/intelmq/harmonization.conf
+/etc/intelmq/BOTS
+/etc/intelmq/pipeline.conf
+/etc/intelmq/defaults.conf
+/etc/intelmq/runtime.conf

debian/control

@@ -2,22 +2,21 @@ Source: intelmq
 Maintainer: Sascha Wilde <[email protected]>
 Section: python
 Priority: optional
-Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt
-X-Python3-Version: >= 3.3.0
+Build-Depends: debhelper (>= 4.1.16), python3-all, python3-setuptools, quilt, dh-python, python-setuptools, dh-systemd, safe-rm, python3-requests, python3-redis, python3-dnspython, python3-psutil, python3-dateutil, python3-termstyle, python3-tz, lsb-release
+X-Python3-Version: >= 3.3
 Standards-Version: 3.9.6
 Homepage: https://github.com/certtools/intelmq/
 
 Package: intelmq
 Architecture: all
 Depends: ${misc:Depends}, ${python3:Depends},
- libffi-dev, python3-dateutil (>= 2.0), python3-dnspython (>= 1.11.1),
- python3-pymongo, python3-openssl, python3-psutil (>= 1.2.1),
- python3-psycopg2, python3-redis (>= 2.10.3),
+ python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1),
+ python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10),
  python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz,
- redis-server, cron, bash-completion, jq
-Recommends: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0),
+ redis-server, cron, bash-completion, jq, systemd
+Suggests: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0),
  python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1),
- python3-geoip2 (>= 2.2.0)
+ python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2
 Description: IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse
  departments,...) for collecting and processing security feeds (such as log
  files) using a message queuing protocol. It's a community driven initiative

debian/cron.d/intelmq-update-data

@@ -3,10 +3,10 @@
 #
 # m h  dom mon dow      command
 # Update data for tor_nodes bot:
-11 0 * * *     intelmq   /usr/bin/update-tor-nodes /opt/intelmq/var/lib/bots/tor_nodes/tor_nodes.dat
+11 0 * * *     intelmq   /usr/bin/update-tor-nodes /var/lib/intelmq/bots/tor_nodes/tor_nodes.dat
 # Update data for maxmind_geoip bot:
-17 0 * * *     intelmq   /usr/bin/update-geoip-data /opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb
+17 0 * * *     intelmq   /usr/bin/update-geoip-data /var/lib/intelmq/bots/maxmind_geoip/GeoLite2-City.mmdb
 # Update data for asn_lookup bot:
-23 0 * * *     intelmq   /usr/bin/update-asn-data /opt/intelmq/var/lib/bots/asn_lookup/ipasn.dat
+23 0 * * *     intelmq   /usr/bin/update-asn-data /var/lib/intelmq/bots/asn_lookup/ipasn.dat
 # Update data for the RIPE DB abuse_c offline contact lookup
-25 6 * * *     intelmq   /usr/bin/update-ripencc_abuse_contact_offline /opt/intelmq/var/lib/bots/ripencc_abuse_contact_offline/
+25 6 * * *     intelmq   /usr/bin/update-ripencc_abuse_contact_offline /var/lib/intelmq/bots/ripencc_abuse_contact_offline/

debian/intelmq.postinst

@@ -2,11 +2,15 @@
 set -e
 
 if [ "$1" = "configure" ] ; then
+    if ! getent group intelmq >/dev/null 2>&1; then
+        groupadd -r intelmq
+    fi
     if ! getent passwd intelmq >/dev/null 2>&1; then
-        useradd -d /opt/intelmq -U -s /bin/bash intelmq
+        useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
     fi
-    chmod -R g+w /opt/intelmq/*
-    chown -R intelmq:intelmq /opt/intelmq
+    chmod -R g+w /var/lib/intelmq/*
+    chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/
+    systemd-tmpfiles --create /usr/lib/tmpfiles.d/intelmq.conf
 fi
 
 #DEBHELPER#

debian/intelmq.postrm

@@ -2,7 +2,12 @@
 set -e
 
 if [ "$1" = "remove" ] ; then
-  deluser intelmq
+    if getent group intelmq >/dev/null 2>&1; then
+        groupdel -r intelmq
+    fi
+    if getent passwd intelmq >/dev/null 2>&1; then
+        userdel intelmq
+    fi
 fi
 
 #DEBHELPER#

debian/intelmq.tmpfile

@@ -0,0 +1 @@
+D	/run/intelmq	0755	intelmq	intelmq

debian/patches/fix-dnspython-name.patch

@@ -2,7 +2,7 @@ Description: Undo rename of the dependency dnspython3
  The packages was renamed upstream; however, this change does not immediately
  affect this package's target distributions.
 Author: Gernot Schulz <[email protected]>
-Last-Update: 2016-07-20
+Last-Update: 2016-10-04
 ---
 This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
 --- a/setup.py

debian/patches/fix-intelmq-paths.patch

@@ -0,0 +1,24 @@
+Description: Set all paths to LSB
+Author: Sebastian Wagner <[email protected]>
+Last-Update: 2017-09-20
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/intelmq/__init__.py
++++ b/intelmq/__init__.py
+@@ -1,13 +1,12 @@
+ from .version import __version__
+ import os
+
+-ROOT_DIR = "/opt/intelmq/"
+-CONFIG_DIR = os.path.join(ROOT_DIR, "etc/")
++CONFIG_DIR = "/etc/intelmq/"
+ DEFAULT_LOGGING_LEVEL = "INFO"
+ BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS")
+-DEFAULT_LOGGING_PATH = os.path.join(ROOT_DIR, "var/log/")
++DEFAULT_LOGGING_PATH = "/var/log/intelmq/"
+ DEFAULTS_CONF_FILE = os.path.join(CONFIG_DIR, "defaults.conf")
+ HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
+ PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf")
+ RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf")
+-VAR_RUN_PATH = os.path.join(ROOT_DIR, "var/run/")
++VAR_RUN_PATH = "/var/run/intelmq/"

debian/patches/series

@@ -1,2 +1,2 @@
 fix-logrotate-path.patch
-fix-dnspython-name.patch
+fix-intelmq-paths.patch

debian/rules

@@ -1,34 +1,60 @@
 #!/usr/bin/make -f
 
+CODENAME := $(shell lsb_release -cs)
 DOCS_DIR := debian/intelmq/usr/share/doc/intelmq
+export INTELMQ_SKIP_REDIS=1
+export INTELMQ_SKIP_INTERNET=1
 
 # Compile list of bot READMEs
 BOTROOT := intelmq/bots
 BOTCATEGORIES := collectors experts outputs parsers
 BOTCATEGORIES := $(foreach cat,$(BOTCATEGORIES),$(BOTROOT)/$(cat))
-BOTS    := $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*))
+BOTS	:= $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*))
 BOTDOCS := $(foreach bot,$(BOTS),$(wildcard $(bot)/*.md))
 
 # This file is based on a version
 # automatically generated by stdeb 0.8.5 at
 # Wed, 23 Mar 2016 17:49:26 +0000
 export PYBUILD_NAME=intelmq
 %:
-	dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt
+	dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd
+
+build:
+	if [ $(CODENAME) = 'xenial' ] || [ $(CODENAME) = 'jessie' ]; then\
+		patch -p1 setup.py debian/patches/fix-dnspython-name.patch;\
+	fi
+	dh build --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd
 
 override_dh_auto_build:
 
 override_dh_auto_install: $(BOTDOCS)
+	rm intelmq/bin/rewrite_config_files.py
+	rm intelmq/bin/intelmq_gen_harm_docs.py
+	sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py
+	find . -type f -exec sed -i -f debian/sedfile {} \;
 	python3 setup.py install --root=debian/intelmq --prefix=/usr
-	mv debian/intelmq/opt/intelmq/etc/examples/* debian/intelmq/opt/intelmq/etc/
-	rmdir debian/intelmq/opt/intelmq/etc/examples
-	mkdir -p debian/intelmq/opt/intelmq/var/log
-	mkdir -p debian/intelmq/opt/intelmq/var/lib/bots/file-output
+	mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/
+	rmdir debian/intelmq/etc/intelmq/examples
+	# these are already in /usr/bin/
+	#rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/maxmind_geoip/update-geoip-data
+	#rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/asn_lookup/update-asn-data
+	#rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/tor_nodes/update-tor-nodes
+	# and rename those in /usr/bin
+	mv debian/intelmq/usr/bin/update-geoip-data debian/intelmq/usr/bin/intelmq-update-geoip-data
+	mv debian/intelmq/usr/bin/update-asn-data debian/intelmq/usr/bin/intelmq-update-asn-data
+	mv debian/intelmq/usr/bin/update-tor-nodes debian/intelmq/usr/bin/intelmq-update-tor-nodes
+	# create directories
+	mkdir -p debian/intelmq/var/log/intelmq
+	mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output
 	mkdir -p debian/intelmq/etc/logrotate.d
+	mkdir -p debian/intelmq/var/lib/intelmq/bots/modify/
 	## BOTS
 	# Include all bot READMEs
 	for readme in $(foreach bot,$(BOTDOCS),$(subst intelmq/bots/,,$(bot))); \
 	do \
 		mkdir -p $(DOCS_DIR)/bots/$$(dirname $$readme); \
 		cp intelmq/bots/$$readme $(DOCS_DIR)/bots/$$(dirname $$readme); \
 	done
+
+override_dh_systemd_start:
+	dh_systemd_start --restart-after-upgrade

debian/sedfile

@@ -0,0 +1,5 @@
+s/opt\/intelmq\/var\/run/var\/run\/intelmq/g
+s/opt\/intelmq\/var\/log/var\/log\/intelmq/g
+s/opt\/intelmq\/var\/lib/var\/lib\/intelmq/g
+s/opt\/intelmq\/etc\//etc\/intelmq\//g
+s/opt\/intelmq/etc\/intelmq/g

debian/source/format

@@ -1 +1 @@
-3.0 (native)
+3.0 (quilt)

docs/INSTALL.md

@@ -111,11 +111,11 @@ https://software.opensuse.org/download.html?project=home%3Asebix%3Aintelmq&packa
 
 Currently, these operating systems are supported by the packages:
 * CentOS 7, install `epel-release` first
-* Debian 8, install `python3-typing` too
-* Fedora 25
-* openSUSE Leap 42.2 and 42.3
-* openSUSE Tumbleweed
-* Ubuntu 16.04
+* RHEL 7, install `epel-release` first
+* Debian 8 (install `python3-typing` too) and 9
+* Fedora 25, 26 and Rawhide
+* openSUSE Leap 42.2, 42.3 and Tumbleweed
+* Ubuntu 16.04 and 17.04
 
 Please report any errors or improvements at https://github.com/certtools/intelmq/issues Thanks!
 
@@ -158,6 +158,7 @@ sudo -s
 
 pip3 install .
 
+mkdir /opt/intelmq
 useradd -d /opt/intelmq -U -s /bin/bash intelmq
 chmod -R 0770 /opt/intelmq
 chown -R intelmq.intelmq /opt/intelmq

intelmq/bots/collectors/mail/collector_mail_url.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import re
 import io
+import imaplib
 import requests
 
 try:
@@ -26,11 +27,15 @@ def init(self):
         self.chunk_replicate_header = getattr(self.parameters,
                                               'chunk_replicate_header', None)
 
-    def process(self):
+    def connect_mailbox(self):
         mailbox = imbox.Imbox(self.parameters.mail_host,
                               self.parameters.mail_user,
                               self.parameters.mail_password,
                               self.parameters.mail_ssl)
+        return mailbox
+
+    def process(self):
+        mailbox = self.connect_mailbox()
         emails = mailbox.messages(folder=self.parameters.folder, unread=True)
 
         if emails:
@@ -90,7 +95,12 @@ def process(self):
                         # Only mark read if message relevant to this instance,
                         # so other instances watching this mailbox will still
                         # check it.
-                        mailbox.mark_seen(uid)
+                        try:
+                            mailbox.mark_seen(uid)
+                        except imaplib.abort:
+                            # Disconnect, see https://github.com/certtools/intelmq/issues/852
+                            mailbox = self.connect_mailbox()
+                            mailbox.mark_seen(uid)
 
                 if not erroneous:
                     self.logger.info("Email report read.")

intelmq/bots/experts/maxmind_geoip/README.md

@@ -8,3 +8,6 @@
 ```
     "database": "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb"
 ```
+
+This product includes GeoLite2 data created by MaxMind, available from
+<a href="http://www.maxmind.com">http://www.maxmind.com</a>.

intelmq/bots/experts/maxmind_geoip/expert.py

@@ -1,4 +1,8 @@
 # -*- coding: utf-8 -*-
+"""
+This product includes GeoLite2 data created by MaxMind, available from
+<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+"""
 
 from intelmq.lib.bot import Bot