diff --git a/Watcher/README.md b/Watcher/README.md index 2d60766..c7b15d9 100755 --- a/Watcher/README.md +++ b/Watcher/README.md @@ -20,8 +20,8 @@ Please wait until you see: watcher | Performing system checks... watcher | watcher | System check identified no issues (0 silenced). - watcher | October 08, 2020 - 10:28:02 - watcher | Django version 3.1.1, using settings 'watcher.settings' + watcher | October 08, 2022 - 10:28:02 + watcher | Django version 4.1.4, using settings 'watcher.settings' watcher | Starting development server at http://0.0.0.0:9002/ watcher | Quit the server with CONTROL-C. @@ -63,7 +63,7 @@ Most of the settings can be modified from the `/admin` page. There are other settings located in the `.env` file that you can configure. -##### Production Settings [Important] +### Production Settings [Important] In production please put DJANGO_DEBUG environment variable to **False** in the `.env` file: @@ -84,7 +84,7 @@ If you have modified some of these parameters, don't forget to restart all conta docker-compose down docker-compose up -##### Access Watcher remotely within your server instance +### Access Watcher remotely within your server instance In case of **"Bad Request" Error** when accessing Watcher web interface, fill `ALLOWED_HOST` variable (in `.env` file) with your Watcher Server Instance **IP** / or your **FQDN**. It is limited to a **single IP address** / **single FQDN**. @@ -98,7 +98,7 @@ Now, you can restart your instance and the parameters will be taken into account docker-compose down docker-compose up -##### SMTP Server Settings (Email Notifications) +### SMTP Server Settings (Email Notifications) In the `.env` file: EMAIL_FROM=watcher@example.com @@ -113,7 +113,7 @@ Now, you can restart your instance and the parameters will be taken into account docker-compose down docker-compose up -##### TheHive Settings +### TheHive Settings If you want to use **TheHive export**, please fill the **IP** of your TheHive instance and a **generated API key**. In the `.env` file: @@ -128,7 +128,7 @@ Now, you can restart your instance and the parameters will be taken into account docker-compose down docker-compose up -##### MISP Settings +### MISP Settings If you want to use **MISP export**, please fill the **IP** of your MISP instance and an **API key**. In the `.env` file: @@ -143,7 +143,7 @@ Now, you can restart your instance and the parameters will be taken into account docker-compose down docker-compose up -##### LDAP Settings +### LDAP Settings You can configure an LDAP authentication within Watcher: In the `.env` file: diff --git a/Watcher/Watcher/data_leak/core.py b/Watcher/Watcher/data_leak/core.py index f72d632..c6ebb4e 100644 --- a/Watcher/Watcher/data_leak/core.py +++ b/Watcher/Watcher/data_leak/core.py @@ -5,6 +5,7 @@ from django.utils import timezone from datetime import timedelta from apscheduler.schedulers.background import BackgroundScheduler +import tzlocal from django.conf import settings from django.db.models.functions import Length from .mail_template.default_template import get_template @@ -21,7 +22,7 @@ def start_scheduler(): - Fire main every 5 minutes from Monday to Sunday - Fire cleanup every 2 hours """ - scheduler = BackgroundScheduler() + scheduler = BackgroundScheduler(timezone=str(tzlocal.get_localzone())) scheduler.add_job(main_data_leak, 'cron', day_of_week='mon-sun', minute='*/5', id='week_job', max_instances=10, replace_existing=True) diff --git a/Watcher/Watcher/dns_finder/core.py b/Watcher/Watcher/dns_finder/core.py index e0be06e..b858d1a 100644 --- a/Watcher/Watcher/dns_finder/core.py +++ b/Watcher/Watcher/dns_finder/core.py @@ -1,3 +1,4 @@ +# coding=utf-8 import os import six import subprocess @@ -8,6 +9,7 @@ from .mail_template.default_template_cert_transparency import get_cert_transparency_template from .mail_template.group_template import get_group_template from apscheduler.schedulers.background import BackgroundScheduler +import tzlocal from .models import Alert, DnsMonitored, DnsTwisted, Subscriber, KeywordMonitored from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -21,7 +23,7 @@ def start_scheduler(): - Fire main_dns_twist from Monday to Sunday: every 2 hours. - Fire main_certificate_transparency from Monday to Sunday: every hour. """ - scheduler = BackgroundScheduler() + scheduler = BackgroundScheduler(timezone=str(tzlocal.get_localzone())) scheduler.add_job(main_dns_twist, 'cron', day_of_week='mon-sun', hour='*/2', id='main_dns_twist', max_instances=10, replace_existing=True) diff --git a/Watcher/Watcher/site_monitoring/core.py b/Watcher/Watcher/site_monitoring/core.py index 7beb2c1..c840ddc 100644 --- a/Watcher/Watcher/site_monitoring/core.py +++ b/Watcher/Watcher/site_monitoring/core.py @@ -1,6 +1,5 @@ # coding=utf-8 from __future__ import unicode_literals - import smtplib from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart @@ -9,6 +8,7 @@ from django.utils import timezone from datetime import timedelta from apscheduler.schedulers.background import BackgroundScheduler +import tzlocal from .models import Site, Alert, Subscriber import tlsh import requests @@ -31,7 +31,7 @@ def start_scheduler(): Launch multiple planning tasks in background: - Fire monitoring_check from Monday to Sunday : minute='*/6' """ - scheduler = BackgroundScheduler() + scheduler = BackgroundScheduler(timezone=str(tzlocal.get_localzone())) scheduler.add_job(monitoring_check, 'cron', day_of_week='mon-sun', minute='*/6', id='weekend_job', max_instances=10, diff --git a/Watcher/Watcher/threats_watcher/core.py b/Watcher/Watcher/threats_watcher/core.py index 4a41cff..02c1a13 100644 --- a/Watcher/Watcher/threats_watcher/core.py +++ b/Watcher/Watcher/threats_watcher/core.py @@ -6,6 +6,7 @@ from datetime import datetime import calendar from apscheduler.schedulers.background import BackgroundScheduler +import tzlocal from nltk.tokenize import word_tokenize from .mail_template.default_template import get_template import feedparser @@ -24,7 +25,7 @@ def start_scheduler(): - Fire main_watch at 18h00 on Saturday - Fire cleanup every day at 8 am """ - scheduler = BackgroundScheduler() + scheduler = BackgroundScheduler(timezone=str(tzlocal.get_localzone())) scheduler.add_job(main_watch, 'cron', day_of_week='mon-sun', minute='*/30', id='main_watch_job', max_instances=10, @@ -104,8 +105,11 @@ def fetch_last_posts(nb_max_post): posts_published = dict() for url in rss_urls: try: - feed_content = requests.get(url, timeout=60) - feeds.append(feedparser.parse(feed_content.text)) + feed_content = requests.get(url, timeout=10) + if feed_content.status_code == 200: + feeds.append(feedparser.parse(feed_content.text)) + else: + print(str(timezone.now()) + " - " + "Feed: " + url + " => Error: Status code: ", str(feed_content.status_code)) except requests.exceptions.RequestException as e: print(str(timezone.now()) + " - ", e) for feed in feeds: @@ -114,7 +118,10 @@ def fetch_last_posts(nb_max_post): if count <= nb_max_post: count += 1 if 'published_parsed' in post: - dt = datetime.fromtimestamp(calendar.timegm(post.published_parsed)) + if post.published_parsed is not None: + dt = datetime.fromtimestamp(calendar.timegm(post.published_parsed)) + else: + dt = "no-date" else: dt = "no-date" if 'link' in post: diff --git a/Watcher/Watcher/threats_watcher/datas/sources.csv b/Watcher/Watcher/threats_watcher/datas/sources.csv index 6346241..a5409c6 100644 --- a/Watcher/Watcher/threats_watcher/datas/sources.csv +++ b/Watcher/Watcher/threats_watcher/datas/sources.csv @@ -54,44 +54,45 @@ http://10.10.10.7/?action=display&bridge=Twitter&context=By+username&u=CVEnew&no http://10.10.10.7/?action=display&bridge=Twitter&context=By+username&u=meadowmttech&norep=on&nopic=on&noimg=on&noimgscaling=on&format=Mrss http://10.10.10.7/?action=display&bridge=Twitter&context=By+username&u=TheHackersNews&norep=on&nopic=on&noimg=on&noimgscaling=on&format=Mrss http://10.10.10.7/?action=display&bridge=Twitter&context=By+username&u=CSAsingapore&norep=on&nopic=on&noimg=on&noimgscaling=on&format=Mrss +http://10.10.10.7/?action=display&bridge=Twitter&context=By+username&u=CERTEU&norep=on&nopic=on&noimg=on&noimgscaling=on&format=Mrss https://krebsonsecurity.com/feed/ https://www.bleepingcomputer.com/feed/ https://securelist.com/feed/ https://www.silicon.fr/feed -https://www.itespresso.fr/feed https://korben.info/feed https://www.lemonde.fr/pixels/rss_full.xml https://www.nextinpact.com/rss/news.xml https://www.cert.ssi.gouv.fr/alerte/feed/ https://www.cert.ssi.gouv.fr/ioc/feed/ https://www.cert.ssi.gouv.fr/cti/feed/ -https://www.us-cert.gov/ncas/alerts.xml -https://cert.europa.eu/cert/Data/newsletter/reviewlatest-SecurityBulletins.xml +https://www.cisa.gov/uscert/ncas/alerts.xml https://feeds.feedburner.com/tenable/qaXL?format=xml https://www.fireeye.com/blog/threat-research/_jcr_content.feed -https://ics-cert.us-cert.gov/advisories/advisories.xml +https://www.cisa.gov/uscert/ics/advisories/advisories.xml https://feeds.feedburner.com/feedburner/Talos?format=xml -https://feeds.feedburner.com/ZDI-Upcoming-Advisories https://alas.aws.amazon.com/alas.rss https://access.redhat.com/blogs/766093/feed https://www.debian.org/security/dsa https://feeds.feedburner.com/VmwareSecurityComplianceBlog -https://www.linuxsecurity.com/static-content/linuxsecurity_advisories.rss -https://feedity.com/trends24-in/WltXU1NS.rss -https://www.kaspersky.com/blog/category/threats/feed +https://linuxsecurity.com/linuxsecurity_articles.xml +https://www.kaspersky.com/blog/category/threats/feed/ https://isc.sans.edu/rssfeed.xml https://feeds.feedburner.com/TheHackersNews?format=xml -https://www.theregister.co.uk/security/headlines.atom -https://www.kb.cert.org/vuls/atomfeed -https://www.cyber.gov.au/rssfeed/2 +https://www.theregister.com/security/headlines.atom +https://www.kb.cert.org/vuls/atomfeed/ +https://www.cyber.gov.au/acsc/view-all-content/advisories/rss +https://www.cyber.gov.au/acsc/view-all-content/alerts/rss +https://www.cyber.gov.au/acsc/view-all-content/threats/rss https://www.jpcert.or.jp/english/rss/jpcert-en.rdf -"https://www.welivesecurity.com/category/malware,password,android-2,mobile-security,windows,mac,banking,linux/feed/" -https://securityaffairs.co/wordpress/feed/ +https://www.welivesecurity.com/category/malware,password,android-2,mobile-security,windows,mac,banking,linux/feed/ +https://www.welivesecurity.com/feed/ +https://securityaffairs.co/feed https://www.mcafee.com/blogs/feed/ https://nakedsecurity.sophos.com/feed/ https://www.zdnet.com/blog/security/rss.xml https://mcafee.com/blogs/feed -https://www.darkreading.com/rss_simple.asp +https://www.darkreading.com/rss.xml https://www.schneier.com/feed/atom/ https://www.incibe-cert.es/feed/avisos-seguridad/all https://www.ccn-cert.cni.es/component/obrss/rss-ultimas-vulnerabilidades.feed +https://www.cyberveille-sante.gouv.fr/alertes-et-vulnerabilites/rss.xml \ No newline at end of file diff --git a/Watcher/docs/_build/doctrees/README.doctree b/Watcher/docs/_build/doctrees/README.doctree index ad59c1b..57ccbce 100644 Binary files a/Watcher/docs/_build/doctrees/README.doctree and b/Watcher/docs/_build/doctrees/README.doctree differ diff --git a/Watcher/docs/_build/doctrees/environment.pickle b/Watcher/docs/_build/doctrees/environment.pickle index 31c9d72..8bfbbc8 100644 Binary files a/Watcher/docs/_build/doctrees/environment.pickle and b/Watcher/docs/_build/doctrees/environment.pickle differ diff --git a/Watcher/docs/_build/doctrees/index.doctree b/Watcher/docs/_build/doctrees/index.doctree index 85b6b61..5a97f2c 100644 Binary files a/Watcher/docs/_build/doctrees/index.doctree and b/Watcher/docs/_build/doctrees/index.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/data_leak_core.doctree b/Watcher/docs/_build/doctrees/modules/data_leak_core.doctree index 8d7220d..45b345d 100644 Binary files a/Watcher/docs/_build/doctrees/modules/data_leak_core.doctree and b/Watcher/docs/_build/doctrees/modules/data_leak_core.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/dns_finder_core.doctree b/Watcher/docs/_build/doctrees/modules/dns_finder_core.doctree index c38cc8e..9a254d7 100644 Binary files a/Watcher/docs/_build/doctrees/modules/dns_finder_core.doctree and b/Watcher/docs/_build/doctrees/modules/dns_finder_core.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/site_monitoring_core.doctree b/Watcher/docs/_build/doctrees/modules/site_monitoring_core.doctree index 064d524..da87d1e 100644 Binary files a/Watcher/docs/_build/doctrees/modules/site_monitoring_core.doctree and b/Watcher/docs/_build/doctrees/modules/site_monitoring_core.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/site_monitoring_misp.doctree b/Watcher/docs/_build/doctrees/modules/site_monitoring_misp.doctree index 315d83c..9f35c58 100644 Binary files a/Watcher/docs/_build/doctrees/modules/site_monitoring_misp.doctree and b/Watcher/docs/_build/doctrees/modules/site_monitoring_misp.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/site_monitoring_thehive.doctree b/Watcher/docs/_build/doctrees/modules/site_monitoring_thehive.doctree index 007edaf..3324dda 100644 Binary files a/Watcher/docs/_build/doctrees/modules/site_monitoring_thehive.doctree and b/Watcher/docs/_build/doctrees/modules/site_monitoring_thehive.doctree differ diff --git a/Watcher/docs/_build/doctrees/modules/threats_watcher_core.doctree b/Watcher/docs/_build/doctrees/modules/threats_watcher_core.doctree index 260746b..58898a8 100644 Binary files a/Watcher/docs/_build/doctrees/modules/threats_watcher_core.doctree and b/Watcher/docs/_build/doctrees/modules/threats_watcher_core.doctree differ diff --git a/Watcher/docs/_build/html/.buildinfo b/Watcher/docs/_build/html/.buildinfo index 46031e1..465eddc 100644 --- a/Watcher/docs/_build/html/.buildinfo +++ b/Watcher/docs/_build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 774af62ba3b4c3e7b8aa82edf615a895 +config: 7aab10be9f2d22f004142a4c3d009153 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/Watcher/docs/_build/html/README.html b/Watcher/docs/_build/html/README.html index f090a6d..5a3324d 100644 --- a/Watcher/docs/_build/html/README.html +++ b/Watcher/docs/_build/html/README.html @@ -4,7 +4,7 @@ -