Merge pull request #13 from CybercentreCanada/feature/new_apis

Feature/new apis

assemblyline_client/v4_client/client.py

@@ -10,12 +10,14 @@
 from assemblyline_client.v4_client.module.heuristics import Heuristics
 from assemblyline_client.v4_client.module.ingest import Ingest
 from assemblyline_client.v4_client.module.result import Result
+from assemblyline_client.v4_client.module.safelist import Safelist
 from assemblyline_client.v4_client.module.search import Search
 from assemblyline_client.v4_client.module.service import Service
 from assemblyline_client.v4_client.module.signature import Signature
 from assemblyline_client.v4_client.module.socketio import SocketIO
 from assemblyline_client.v4_client.module.submission import Live, Submission
 from assemblyline_client.v4_client.module.submit import Submit
+from assemblyline_client.v4_client.module.system import System
 from assemblyline_client.v4_client.module.user import User
 from assemblyline_client.v4_client.module.workflow import Workflow
 
@@ -34,12 +36,14 @@ def __init__(self, connection):
         self.ingest = Ingest(self._connection)
         self.live = Live(self._connection)
         self.result = Result(self._connection)
+        self.safelist = Safelist(self._connection)
         self.search = Search(self._connection)
         self.service = Service(self._connection)
         self.signature = Signature(self._connection)
         self.socketio = SocketIO(self._connection)
         self.submission = Submission(self._connection)
         self.submit = Submit(self._connection)
+        self.system = System(self._connection)
         self.user = User(self._connection)
         self.workflow = Workflow(self._connection)
 

assemblyline_client/v4_client/common/utils.py

@@ -12,7 +12,7 @@
 
 
 INVALID_STREAM_SEARCH_PARAMS = ('deep_paging_id', 'rows', 'sort')
-SEARCHABLE = ('alert', 'file', 'heuristic', 'result', 'signature', 'submission', 'workflow')
+SEARCHABLE = ('alert', 'file', 'heuristic', 'result', 'safelist', 'signature', 'submission', 'workflow')
 API = 'v4'
 
 

assemblyline_client/v4_client/module/safelist.py

@@ -0,0 +1,59 @@
+from json import dumps
+
+from assemblyline_client.v4_client.common.utils import api_path
+
+
+class Safelist(object):
+    def __init__(self, connection):
+        self._connection = connection
+
+    def __call__(self, qhash):
+        """\
+Check if a hash exists in the safelist
+
+Required:
+qhash     : Hash to check in the safelist (string)
+"""
+        return self._connection.get(api_path('safelist', qhash))
+
+    def add_update(self, safelist_object):
+        """\
+Add a hash in the safelist if it does not exist or update its list of sources if it does
+
+Required:
+safelist_object     : A dictionary containing the safelist details
+
+Throws a Client exception if the safelist object is invalid.
+"""
+        return self._connection.put(api_path('safelist'), data=dumps(safelist_object))
+
+    def delete(self, safelist_id):
+        """\
+Delete the safelist object using it's ID.
+
+The safelist ID is one of the following:
+    - One of the hash of the file in order of importance (SHA256 -> SHA1 -> MD5)
+    - SHA256 hash of the tag type and value concatenated
+
+Required:
+safelist_id     : ID of the safelist object (string)
+
+Throws a Client exception if the safelist does not exist.
+"""
+        return self._connection.delete(api_path('safelist', safelist_id))
+
+    def set_enabled(self, safelist_id, enabled):
+        """\
+Set the enabled status of a safelist object using it's ID.
+
+The safelist ID is one of the following:
+    - One of the hash of the file in order of importance (SHA256 -> SHA1 -> MD5)
+    - SHA256 hash of the tag type and value concatenated
+
+Required:
+safelist_id     : ID of the safelist object (string)
+enabled         : True/False value (boolean)
+
+Throws a Client exception if the safelist does not exist.
+"""
+        return self._connection.put(api_path('safelist', 'enable', safelist_id), data=dumps(enabled))

assemblyline_client/v4_client/module/search/__init__.py

@@ -19,9 +19,9 @@ def __init__(self, connection):
         self.stats = Stats(connection)
         self.stream = Stream(connection, self._do_search)
 
-    def _do_search(self, bucket, query, **kwargs):
-        if bucket not in SEARCHABLE:
-            raise ClientError("Bucket %s is not searchable" % bucket, 400)
+    def _do_search(self, index, query, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
 
         filters = kwargs.pop('filters', None)
         if filters is not None:
@@ -32,7 +32,7 @@ def _do_search(self, bucket, query, **kwargs):
 
         kwargs = {k: v for k, v in kwargs.items() if v is not None}
         kwargs['query'] = query
-        path = api_path('search', bucket)
+        path = api_path('search', index)
         return self._connection.post(path, data=json.dumps(kwargs))
 
     def alert(self, query, filters=None, fl=None, offset=0, rows=25, sort=None, timeout=None):
@@ -115,6 +115,26 @@ def result(self, query, filters=None, fl=None, offset=0, rows=25, sort=None, tim
         return self._do_search('result', query, filters=filters, fl=fl, offset=offset,
                                rows=rows, sort=sort, timeout=timeout)
 
+    def safelist(self, query, filters=None, fl=None, offset=0, rows=25, sort=None, timeout=None):
+        """\
+Search safelist with a lucene query.
+
+Required:
+query   : lucene query (string)
+
+Optional:
+filters : Additional lucene queries used to filter the data (list of strings)
+fl      : List of fields to return (comma separated string of fields)
+offset  : Offset at which the query items should start (integer)
+rows    : Number of records to return (integer)
+sort    : Field used for sorting with direction (string: ex. 'id desc')
+timeout : Max amount of miliseconds the query will run (integer)
+
+Returns all results.
+"""
+        return self._do_search('safelist', query, filters=filters, fl=fl, offset=offset,
+                               rows=rows, sort=sort, timeout=timeout)
+
     def signature(self, query, filters=None, fl=None, offset=0, rows=25, sort=None, timeout=None):
         """\
 Search signatures with a lucene query.

assemblyline_client/v4_client/module/search/facet.py

@@ -5,9 +5,9 @@ class Facet(object):
     def __init__(self, connection):
         self._connection = connection
 
-    def _do_facet(self, bucket, field, **kwargs):
-        if bucket not in SEARCHABLE:
-            raise ClientError("Bucket %s is not searchable" % bucket, 400)
+    def _do_facet(self, index, field, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
 
         filters = kwargs.pop('filters', None)
         if filters is not None:
@@ -19,7 +19,7 @@ def _do_facet(self, bucket, field, **kwargs):
         kwargs = {k: v for k, v in kwargs.items() if v is not None and k != 'filters'}
         if filters is not None:
             kwargs['params_tuples'] = filters
-        path = api_path('search', 'facet', bucket, field, **kwargs)
+        path = api_path('search', 'facet', index, field, **kwargs)
         return self._connection.get(path)
 
     def alert(self, field, query=None, mincount=None, filters=None):
@@ -86,6 +86,22 @@ def result(self, field, query=None, mincount=None, filters=None):
 """
         return self._do_facet('result', field, query=query, mincount=mincount, filters=filters)
 
+    def safelist(self, field, query=None, mincount=None, filters=None):
+        """\
+List most frequent value for a field in the safelist collection.
+
+Required:
+field   : field to extract the facets from
+
+Optional:
+qeury    : Initial query to filter the data (default: 'id:*')
+filters  : Additional lucene queries used to filter the data (list of strings)
+mincount : Minimum amount of hits for the value to be returned
+
+Returns all results.
+"""
+        return self._do_facet('safelist', field, query=query, mincount=mincount, filters=filters)
+
     def signature(self, field, query=None, mincount=None, filters=None):
         """\
 List most frequent value for a field in the signature collection.

assemblyline_client/v4_client/module/search/fields.py

@@ -5,11 +5,11 @@ class Fields(object):
     def __init__(self, connection):
         self._connection = connection
 
-    def _do_fields(self, bucket):
-        if bucket not in SEARCHABLE:
-            raise ClientError("Bucket %s is not searchable" % bucket, 400)
+    def _do_fields(self, index):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
 
-        path = api_path('search', 'fields', bucket)
+        path = api_path('search', 'fields', index)
         return self._connection.get(path)
 
     def alert(self):
@@ -36,6 +36,12 @@ def result(self):
 """
         return self._do_fields('result')
 
+    def safelist(self):
+        """\
+List all fields details for the safelist collection.
+"""
+        return self._do_fields('safelist')
+
     def signature(self):
         """\
 List all fields details for the signature collection.

assemblyline_client/v4_client/module/search/grouped.py

@@ -6,9 +6,9 @@ class Grouped(object):
     def __init__(self, connection):
         self._connection = connection
 
-    def _do_grouped(self, bucket, field, **kwargs):
-        if bucket not in SEARCHABLE:
-            raise ClientError("Bucket %s is not searchable" % bucket, 400)
+    def _do_grouped(self, index, field, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
 
         filters = kwargs.pop('filters', None)
         if filters is not None:
@@ -20,7 +20,7 @@ def _do_grouped(self, bucket, field, **kwargs):
         kwargs = {k: v for k, v in kwargs.items() if v is not None and k != 'filters'}
         if filters is not None:
             kwargs['params_tuples'] = filters
-        path = api_path('search', 'grouped', bucket, field, **kwargs)
+        path = api_path('search', 'grouped', index, field, **kwargs)
         return self._connection.get(path)
 
     def alert(self, field, group_sort=None, limit=None, query=None, filters=None,
@@ -115,6 +115,29 @@ def result(self, field, group_sort=None, limit=None, query=None, filters=None,
         return self._do_grouped('result', field, group_sort=group_sort, limit=limit, query=query, filters=filters,
                                 offset=offset, rows=rows, sort=sort, fl=fl)
 
+    def safelist(self, field, group_sort=None, limit=None, query=None, filters=None,
+                 offset=None, rows=None, sort=None, fl=None):
+        """\
+Search safelist collection and group result to a given field
+
+Required:
+field   : Field used to group the results
+
+Optional:
+group_sort : Field used for sorting items in the groups with direction (string: ex. 'id desc')
+limit      : Maximum number of items returned per group (integer)
+query      : lucene query (string)
+filters    : Additional lucene queries used to filter the data (list of strings)
+offset     : Offset at which the query items should start (integer)
+rows       : Number of records to return (integer)
+sort       : Field used for sorting with direction (string: ex. 'id desc')
+fl         : List of fields to return (comma separated string of fields)
+
+Returns a generator that transparently and efficiently pages through results.
+"""
+        return self._do_grouped('safelist', field, group_sort=group_sort, limit=limit, query=query, filters=filters,
+                                offset=offset, rows=rows, sort=sort, fl=fl)
+
     def signature(self, field, group_sort=None, limit=None, query=None, filters=None,
                   offset=None, rows=None, sort=None, fl=None):
         """\

assemblyline_client/v4_client/module/search/histogram.py

@@ -5,9 +5,9 @@ class Histogram(object):
     def __init__(self, connection):
         self._connection = connection
 
-    def _do_histogram(self, bucket, field, **kwargs):
-        if bucket not in SEARCHABLE:
-            raise ClientError("Bucket %s is not searchable" % bucket, 400)
+    def _do_histogram(self, index, field, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
 
         filters = kwargs.pop('filters', None)
         if filters is not None:
@@ -19,12 +19,12 @@ def _do_histogram(self, bucket, field, **kwargs):
         kwargs = {k: v for k, v in kwargs.items() if v is not None and k != 'filters'}
         if filters is not None:
             kwargs['params_tuples'] = filters
-        path = api_path('search', 'histogram', bucket, field, **kwargs)
+        path = api_path('search', 'histogram', index, field, **kwargs)
         return self._connection.get(path)
 
     def alert(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the alert bucket where the frequency
+Create an histogram of data from a given field in the alert index where the frequency
 of the data is split between a given gap size.
 
 Required:
@@ -45,7 +45,7 @@ def alert(self, field, query=None, mincount=None, filters=None, start=None, end=
 
     def file(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the file bucket where the frequency
+Create an histogram of data from a given field in the file index where the frequency
 of the data is split between a given gap size.
 
 Required:
@@ -66,7 +66,7 @@ def file(self, field, query=None, mincount=None, filters=None, start=None, end=N
 
     def result(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the result bucket where the frequency
+Create an histogram of data from a given field in the result index where the frequency
 of the data is split between a given gap size.
 
 Required:
@@ -85,9 +85,30 @@ def result(self, field, query=None, mincount=None, filters=None, start=None, end
         return self._do_histogram('result', field, query=query, mincount=mincount, filters=filters,
                                   start=start, end=end, gap=gap)
 
+    def safelist(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
+        """\
+Create an histogram of data from a given field in the safelist index where the frequency
+of the data is split between a given gap size.
+
+Required:
+field   : field to create the histograms with (only work on date or number fields)
+
+Optional:
+query    : Initial query to filter the data (default: 'id:*')
+filters  : Additional lucene queries used to filter the data (list of strings)
+mincount : Minimum amount of hits for the value to be returned
+start    : Beginning of the histogram range (Default: now-1d or 0)
+end      : End of the histogram range (Default: now or 1000)
+gap      : Interval in between each histogram points (Default: 1h or 100)
+
+Returns all results.
+"""
+        return self._do_histogram('safelist', field, query=query, mincount=mincount, filters=filters,
+                                  start=start, end=end, gap=gap)
+
     def signature(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the signature bucket where the frequency
+Create an histogram of data from a given field in the signature index where the frequency
 of the data is split between a given gap size.
 
 Required:
@@ -108,7 +129,7 @@ def signature(self, field, query=None, mincount=None, filters=None, start=None,
 
     def submission(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the submission bucket where the frequency
+Create an histogram of data from a given field in the submission index where the frequency
 of the data is split between a given gap size.
 
 Required:
@@ -129,7 +150,7 @@ def submission(self, field, query=None, mincount=None, filters=None, start=None,
 
     def workflow(self, field, query=None, mincount=None, filters=None, start=None, end=None, gap=None):
         """\
-Create an histogram of data from a given field in the workflow bucket where the frequency
+Create an histogram of data from a given field in the workflow index where the frequency
 of the data is split between a given gap size.
 
 Required: