Quick updates

analyzers/Abuse_Finder/requirements.txt

@@ -1,3 +1,4 @@
 cortexutils
+git+https://github.com/monoidic/ipwhois.git
 abuse_finder
 future

analyzers/Censys/requirements.txt

@@ -1,2 +1,3 @@
 cortexutils
-censys==1.1.1
+argcomplete
+censys==2.2.11

analyzers/MISPWarningLists/mispwarninglists.py

@@ -161,18 +161,19 @@ def run(self):
                         "SELECT list_name, list_version, concat(subdomain, '.', domain, '.', tld) as value  FROM warninglists WHERE (subdomain = '%s' or subdomain = '*') and domain = '%s' and tld = '%s'"
                         % (subdomain, domain, tld)
                     )
-            values = self.engine.execute(sql)
+            with self.engine.connect() as conn:
+                values = conn.execute(db.text(sql))
+                if values.rowcount > 0:
+                    for row in values:
+                        results.append(
+                            {
+                                key: value
+                                for (key, value) in zip(
+                                    ["list_name", "list_version", "value"], row
+                                )
+                            }
+                        )
             self.engine.dispose()
-            if values.rowcount > 0:
-                for row in values:
-                    results.append(
-                        {
-                            key: value
-                            for (key, value) in zip(
-                                ["list_name", "list_version", "value"], row
-                            )
-                        }
-                    )
             self.report({"results": results, "mode": "db", "is_uptodate": "N/A"})
 
     def summary(self, raw):

analyzers/MISPWarningLists/warninglists_create_db.py

@@ -20,7 +20,7 @@
 conn_string = "<insert_postgres_conn_strin>"
 warninglists_path = "misp-warninglists/**/list.json"
 
-engine = create_engine(conn_string, use_batch_mode=True)
+engine = create_engine(conn_string))
 conn = engine.connect()
 
 # UPDATE TLD FROM MOZILLA
@@ -148,7 +148,10 @@
 
 
 # CHECK IF OLD RELEASE ARE IN DB
-s = select([warninglists.c.list_name, warninglists.c.list_version]).distinct()
+try:
+    s = select([warninglists.c.list_name, warninglists.c.list_version]).distinct()
+except sqlalchemy.exc.ArgumentError:
+    s = select(warninglists.c.list_name, warninglists.c.list_version).distinct()
 last_versions = [x for x in conn.execute(s)]
 print(f"{len(last_versions)} list already available in db")
 

analyzers/MalwareClustering/requirements.txt

@@ -3,5 +3,5 @@ requests
 pyimpfuzzy==0.5
 # py2neo is EOL and older versions were deleted from pipy https://github.com/neo4j-contrib/py2neo
 py2neo==2021.2.4
-apiscout==1.1.5
-python-magic==0.4.22
+apiscout
+python-magic==0.4.27

analyzers/Virusshare/getHashes.sh

@@ -3,7 +3,7 @@
 
 
 display_usage() { 
-    echo "getHashes v0.2"
+    echo "getHashes v0.3"
     echo "  Fetch all Virusshare.com hashes" 
     echo -e "\n  Usage: $0 <path> \n"
 } 
@@ -20,17 +20,19 @@ if [ ! -d $1 ]; then
 
 fi
 
-cd $1
-for u in `curl https://virusshare.com/hashes.4n6 | grep -E "VirusShare_[0-9]{5}\.md5" | c\
-ut -d\" -f2 | cut -d\/ -f2`
+WD=$1
+declare -a base_urls=($(printf 'url=https://virusshare.com/hashfiles/%0.s\n' {1..1}))
+declare -a base_outs=($(printf 'output=./%0.s\n' {1..1}))
+
+pushd $WD
+while mapfile -t -n 8 ary && ((${#ary[@]}));
 do
-    echo $u
-    if [ -e $1/$u ]; then
-        echo "File already downloaded"
-    else
-        wget https://virusshare.com/hashes/$u
-        sleep 3
-    fi
-
-done | tee -a ../$0.log
-cd ..
+  rm -f ../config
+  IFS=,
+  eval echo "${base_urls[*]}"{"${ary[*]}"} | tr " " "\n" >> ../config
+  eval echo "${base_outs[*]}"{"${ary[*]}"} | tr " " "\n" >> ../config
+  curl -s -N --parallel --parallel-immediate --parallel-max 8 --config config | tee -a ../$0.log
+  sleep 3
+done <<< `curl -s -L https://virusshare.com/hashes.4n6 | grep -E "VirusShare_[0-9]{5}\.md5" | cut -d\" -f2 | cut -d\/ -f2`
+popd
+