diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..eb5ced2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: python +python: +- '3.6' +install: +- sudo apt-get install pandoc +- pip install -r requirements.txt +script: +#- python -m pytest +- python setup.py bdist_wheel --universal +deploy: + provider: pypi + user: "thehive-project" + password: + secure: UFT+5CY4uqrCuZJvm4wbAyY7XEKcDz//VSt+jGPktj2/PxnmL7Qj5EA+2BFwOpLwUpZw3eWHPZrLkquDYsuR+BtXK08QVxtYHZJiNLdc+bM+R8UQh+VSuu4IQYqtUMVBKZtMkkx8ss+LgAdB+ArUKfVn5HVOOmEV8D4Ghx1Yf90D3zBrDfu6i/h3OajNgSrSdy6i/B7EyIjqZ5rfffCroxl9jPvWu8kPimaknRav6qDFykT4golJGoe64IUEz5AnuhbyBc1VTXCOKcjXCaYj6VSfXFxQVZz/vO+DGsFajybDyYwts6z5GD9kx9GFwNhUDVtDoEybMaY1a1UwBZi9OPG/fmUv4M7qQ5yh9YgByhw3B20JElgfgGsOSvmXIZhw9lAhkvRSPom64HIWRFZCtMMEH3f5gzwite07rcsCfV+VDypNa5eOkAKnFg21p2ibG+fij7bpajwnMxiZf3KMpW4F5D25MAu7Rf3+dyfoZj7sA0ElEdzUTbMAZHTST1Zk2CCyoE69PNuPt6ZTmv9oDgWd5GreXfyw4pP/ehR5VDRG/eNv1hzp1Mg328IZhFcS7wwaCb8yh4ZHq4uUF4Egsmx+IhvqXgtrLHKEW9t5ndS+Z7oe+EKU1sLlCFPqzNFVPmqWotZO4gHQ6cF3due6AZnAGlBE69rIkmPrtR8rsW0= + distributions: "bdist_wheel" + on: + tags: true \ No newline at end of file diff --git a/README.md b/README.md index 7c09e96..2e647f1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,19 @@ -[![Join the chat at https://gitter.im/TheHive-Project/TheHive](https://badges.gitter.im/TheHive-Project/TheHive.svg)](https://gitter.im/TheHive-Project/TheHive?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - +
+

+ + Build status + + + Discord + + + License + + + Pypi page + +

+
# Cortex4py Cortex4py is a Python API client for [Cortex](https://thehive-project.org/), a powerful observable analysis engine where observables such as IP and email addresses, URLs, domain names, files or hashes can be analyzed one by one using a Web interface. @@ -44,7 +58,7 @@ We welcome your contributions. Please feel free to fork the code, play with it, We do have a [Code of conduct](code_of_conduct.md). Make sure to check it out before contributing. # Support -Please [open an issue on GitHub](https://github.com/CERT-BDF/Cortex4py/issues/new) if you'd like to report a bug or request a feature. We are also available on [Gitter](https://gitter.im/TheHive-Project/TheHive) to help you out. +Please [open an issue on GitHub](https://github.com/TheHive-Project/Cortex4py/issues/new) if you'd like to report a bug or request a feature. We are also available on [Discord](https://chat.thehive-project.org) to help you out. If you need to contact the project team, send an email to . diff --git a/Usage.md b/Usage.md index 294804c..4123d65 100644 --- a/Usage.md +++ b/Usage.md @@ -24,10 +24,14 @@ Cortex4py 2 requires Python 3. It does not work with Cortex 1.x. * [Model](#model-2) * [Methods](#methods-2) * [Examples](#examples-2) -* [Job operations](#job-operations) +* [Responder operations](#responder-operations) * [Model](#model-3) * [Methods](#methods-3) * [Examples](#examples-3) +* [Job operations](#job-operations) + * [Model](#model-4) + * [Methods](#methods-4) + * [Examples](#examples-4) ## Introduction @@ -226,7 +230,7 @@ org = api.organizations.get_by_id('demo') print(json.dumps(org.json(), indent=2)) # Fetch the last 5 created and active users -users = api.organizations.get_users(org.id, Eq('status', 'Active'), range='0-5', sort='-createdAt') +users = api.organizations.get_users(org.id, Eq('status', 'Ok'), range='0-5', sort='-createdAt') # Display the usernames for user in users: @@ -392,6 +396,7 @@ An analyzer is represented by the following model class: | `dataTypeList` | Allowed datatypes | readonly | | `baseConfig` | Base configuration name. This identifies the shared set of configuration with all the analyzer's flavors | readonly | | `jobCache` | Report cache timeout in minutes, visible for `orgAdmin` users only | writable | +| `jobTimeout` | Job timeout in minutes, visible for `orgAdmin` users only | writable | | `rate` | Numeric amount of analyzer calls authorized for the specified `rateUnit`, visible for `orgAdmin` users only | writable | | `rateUnit` | Period of availability of the rate limite: `Day` or `Month`, visible for `orgAdmin` users only | writable | | `configuration` | A JSON object where key/value pairs represent the config names, and their values. It includes the default properties `proxy_http`, `proxy_https`, `auto_extract_artifacts`, `check_tlp`, and `max_tlp`, visible for `orgAdmin` users only | writable | @@ -445,16 +450,18 @@ analyzer = api.analyzers.enable('Test_1_0', { "proxy_https": "http://localhost:9999", "auto_extract_artifacts": False, "check_tlp": True, - "max_tlp": 2 + "max_tlp": 2, + "max_pap": 2 }, + "jobCache": 10, + "jobTimeout": 30, "rate": 1000, - "rateUnit": "Day", - "jobCache": 5 + "rateUnit": "Day" }) # Print the details of the enaled analyzer print(json.dumps(analyzer.json(), indent=2)) -print(analyzer.analyzerDefinitionId == 'Test_1_0') +print(analyzer.workerDefinitionId == 'Test_1_0') # Update the configuration analyzer_id = analyzer.id @@ -468,7 +475,8 @@ analyzer = api.analyzers.update(analyzer.id, { "proxy_https": null, "auto_extract_artifacts": True, "check_tlp": false, - "max_tlp": null + "max_tlp": null, + "max_pap": 2 } }) @@ -498,9 +506,135 @@ print(json.dumps(job2.json(), indent=2)) api.analyzers.disable(analyzer_id) ``` +## Responder Operations + +The `RespondersController` class provides a set of methods to handle responders. + +### Model + +A responder is an instance of a responder definition, and both models share the same fields. + +A responder definition is represented by the following model class: + +| Field | Description | Type | +| --------- | ----------- | ---- | +| `id` | Responder ID once enabled within an organization | readonly | +| `workerDefinitionId`| Responder definition name | readonly | +| `name` | Name of the responder | readonly | +| `version` | Version of the responder | readonly | +| `description` | Description of the responder | readonly | +| `author` | Author of the responder | readonly | +| `url` | URL where the responder has been published | readonly | +| `license` | License of the responder | readonly | +| `dataTypeList` | Allowed datatypes | readonly | +| `configurationItems` | A list that describes the configuration options of the responder | readonly | +| `baseConfig` | Base configuration name. This identifies the shared set of configuration with all the responder's flavors | readonly | +| `createdBy` | User who enabled the responder | computed | +| `updatedAt` | Last update date | computed | +| `updatedBy` | User who last updated the responder | computed | + +A responder is represented by the following model class: + +| Field | Description | Type | +| --------- | ----------- | ---- | +| `id` | Responder ID once enabled within an organization | readonly | +| `workerDefinitionId`| Responder definition name | readonly | +| `name` | Name of the responder | readonly | +| `version` | Version of the responder | readonly | +| `description` | Description of the responder | readonly | +| `author` | Author of the responder | readonly | +| `url` | URL where the responder has been published | readonly | +| `license` | License of the responder | readonly | +| `dataTypeList` | Allowed datatypes | readonly | +| `baseConfig` | Base configuration name. This identifies the shared set of configuration with all the responder's flavors | readonly | +| `jobCache` | Report cache timeout in minutes, visible for `orgAdmin` users only | writable | +| `rate` | Numeric amount of responder calls authorized for the specified `rateUnit`, visible for `orgAdmin` users only | writable | +| `rateUnit` | Period of availability of the rate limite: `Day` or `Month`, visible for `orgAdmin` users only | writable | +| `configuration` | A JSON object where key/value pairs represent the config names, and their values. It includes the default properties `proxy_http`, `proxy_https`, `auto_extract_artifacts`, `check_tlp`, and `max_tlp`, visible for `orgAdmin` users only | writable | +| `createdBy` | User who enabled the analyzer | computed | +| `updatedAt` | Last update date | computed | +| `updatedBy` | User who last updated the analyzer | computed | + +### Methods + +| Method | Description | Return type | +| --------- | ----------- | ---- | +|`find_all(query,**kwargs)` | Returns a list of `Responder` objects, based on `query`, `range` and `sort` parameters | List[Responder] | +|`find_one_by(query,**kwargs)` | Returns the first `Responder` object, based on `query` and `sort` parameters | Responder | +|`get_by_id(worker_id)` | Returns a `Responder` by its `id` | Responder | +|`get_by_name(name)` | Returns a `Responder` by its `name` | Responder | +|`get_by_type(data_type)` | Returns a list of available `Responder` applicable to the given `data_type` | List[Responder] | +|`enable(responder_name,config)` | Activate an responder and returns its `Responder` object | Responder | +|`update(worker_id)` | Update the configuration of an `Responder` and returns the updated version | Responder | +|`disable(worker_id)` | Removes a responder from an organization and returns `true` if it completes successfully | Boolean | +|`run_by_id(worker_id, data,**kwargs)` | Returns a `Job` by its `name` | Job | +|`run_by_name(responder_name, data,**kwargs)` | Runs a responder by its name and returns the resulting `Job` | Job | +|`definitions()` | Returns the list of all the responder definitions including the enabled and disabled responders | List[ResponderDefinition] | + +### Examples + +The following example shows how to manipulate responders: + +```python +import json + +from cortex4py.api import Api +from cortex4py.query import * + +api = Api('http://CORTEX_APP_URL:9001', '**API_KEY**') + +# Get enabled responders +responders = api.responders.find_all({}, range='all') + +# Display enabled responders' names +for responder in responders: + print('Responder {} is enabled'.format(responder.name)) + +# Get enabled responders that available for TheHive cases +case_responders = api.responders.get_by_type('thehive:case') + +# Display responders details +for responder in case_responders: + print(json.dumps(responder.json(), indent=2)) + +# Enable the responder called Test_1_0 +responder = api.responders.enable('Test_1_0', { + "configuration": { + "api_key": "XXXXXXXXXXXXXx", + "proxy_http": "http://localhost:9999", + "proxy_https": "http://localhost:9999", + "check_tlp": True, + "max_tlp": 2, + "max_pap": 2 + }, + "jobTimeout": 30, + "rate": 1000, + "rateUnit": "Day" +}) + +# Print the details of the enaled responder +print(json.dumps(responder.json(), indent=2)) +print(responder.workerDefinitionId == 'Test_1_0') + +# Run a responder +job = api.responders.run_by_name('File_Info_2_0', { + 'data': { + 'title': 'Sample case', + 'description': 'This is a sample case', + ... + }, + 'dataType': 'thehive:case', + 'tlp': 1 +}) +print(json.dumps(job.json(), indent=2)) + +# Disable a responder +api.responders.disable(responder.id) +``` + ## Job Operations -The `JobsController` class provides a set of methods to handle jobs. A job is the execution of a specific analyzer. +The `JobsController` class provides a set of methods to handle jobs. A job is the execution of a specific worker (analyzer or responder). ### Model @@ -509,18 +643,19 @@ A job is represented by the following model class: | Attribute | Description | Type | | --------- | ----------- | ---- | | `id` | Job ID | computed | -| `analyzerDefinitionId`| Analyzer definition name | readonly | -| `analyzerId` | Instance ID of the analyzer to which the job is associated | readonly | +| `type` | Job type: `responder` or `analyzer` | computed | +| `workerDefinitionId`| Worker definition name | readonly | +| `workerId` | Instance ID of the worker to which the job is associated | readonly | +| `workerName` | Name of the worker to which the job is associated | readonly | | `organization` | Organization to which the user belongs (set upon account creation) | readonly | -| `analyzerName` | Name of the analyzer to which the job is associated | readonly | -| `dataType` | the datatype of the analyzed observable | readonly | +| `dataType` | the datatype of the worker's input data | readonly | | `status` | Status of the job (`Waiting`, `InProgress`, `Success`, `Failure`, `Deleted`) | computed | -| `data` | Value of the analyzed observable (does not apply to `file` observables) | readonly | +| `data` | Value of the worker's input (does not apply to `file` observables). Contains all the data of a `Case` if the job is a result of a case responder. | readonly | | `attachment` | JSON object representing `file` observables (does not apply to non-`file` observables). It defines the`name`, `hashes`, `size`, `contentType` and `id` of the `file` observable | readonly | | `parameters` | JSON object of key/value pairs set during job creation | readonly | | `message` | A free text field to set additional text/context for a job | readonly | | `tlp` | The TLP of the analyzed observable | readonly | -| `report` | The analysy report as a JSON object including `success`, `full`, `summary` and `artifacts` peoperties.
In case of failure, the resport contains a `errorMessage` property | readonly | +| `report` | The analysis report as a JSON object including `success`, `full`, `summary` and `artifacts` peoperties.
In case of failure, the resport contains a `errorMessage` property | readonly | | `startDate` | Start date | computed | | `endDate` | End date | computed | | `createdAt` | Creation date. Please note that a job can be requested but not immediately honored. The actual time at which it is started is the value of `startDate` | computed | diff --git a/cortex4py/api.py b/cortex4py/api.py index 7a10a0d..cd55e28 100644 --- a/cortex4py/api.py +++ b/cortex4py/api.py @@ -10,6 +10,7 @@ from .controllers.users import UsersController from .controllers.jobs import JobsController from .controllers.analyzers import AnalyzersController +from .controllers.responders import RespondersController class Api(object): @@ -33,6 +34,7 @@ def __init__(self, url, api_key, **kwargs): self.users = UsersController(self) self.jobs = JobsController(self) self.analyzers = AnalyzersController(self) + self.responders = RespondersController(self) @staticmethod def __recover(exception): @@ -151,8 +153,8 @@ def get_analyzers(self, data_type=None): 'api.get_analyzers() is considered deprecated. Use api.analyzers.get_by_[id|name|type]() instead.', DeprecationWarning ) - if data_type is not None: - return self.analyzers.find_all() + if data_type is None: + return self.analyzers.find_all({}) else: return self.analyzers.get_by_type(data_type) diff --git a/cortex4py/controllers/__init__.py b/cortex4py/controllers/__init__.py index d2a14e6..7af71fa 100644 --- a/cortex4py/controllers/__init__.py +++ b/cortex4py/controllers/__init__.py @@ -4,3 +4,4 @@ from .users import UsersController from .jobs import JobsController from .analyzers import AnalyzersController +from .responders import RespondersController diff --git a/cortex4py/controllers/analyzers.py b/cortex4py/controllers/analyzers.py index a223be9..1c1fb9a 100644 --- a/cortex4py/controllers/analyzers.py +++ b/cortex4py/controllers/analyzers.py @@ -7,6 +7,7 @@ from cortex4py.query import * from .abstract import AbstractController from ..models import Analyzer, Job, AnalyzerDefinition +from ..exceptions import CortexError class AnalyzersController(AbstractController): @@ -48,11 +49,13 @@ def disable(self, analyzer_id) -> bool: def run_by_id(self, analyzer_id, observable, **kwargs) -> Job: tlp = observable.get('tlp', 2) + pap = observable.get('pap', 2) data_type = observable.get('dataType', None) post = { 'dataType': data_type, - 'tlp': tlp + 'tlp': tlp, + 'pap': pap } params = {} @@ -85,4 +88,7 @@ def run_by_id(self, analyzer_id, observable, **kwargs) -> Job: def run_by_name(self, analyzer_name, observable, **kwargs) -> Job: analyzer = self.get_by_name(analyzer_name) + if analyzer is None: + raise CortexError("Analyzer %s not found" % analyzer_name) + return self.run_by_id(analyzer.id, observable, **kwargs) diff --git a/cortex4py/controllers/responders.py b/cortex4py/controllers/responders.py new file mode 100644 index 0000000..b355c29 --- /dev/null +++ b/cortex4py/controllers/responders.py @@ -0,0 +1,66 @@ +from typing import List + +from cortex4py.query import * +from .abstract import AbstractController +from ..models import Responder, Job, ResponderDefinition + + +class RespondersController(AbstractController): + def __init__(self, api): + AbstractController.__init__(self, 'responder', api) + + def find_all(self, query, **kwargs) -> List[Responder]: + return self._wrap(self._find_all(query, **kwargs), Responder) + + def find_one_by(self, query, **kwargs) -> Responder: + return self._wrap(self._find_one_by(query, **kwargs), Responder) + + def get_by_id(self, worker_id) -> Responder: + return self._wrap(self._get_by_id(worker_id), Responder) + + def get_by_name(self, name) -> Responder: + return self._wrap(self._find_one_by(Eq('name', name)), Responder) + + def get_by_type(self, data_type) -> List[Responder]: + return self._wrap(self._api.do_get('responder/type/{}'.format(data_type)).json(), Responder) + + def definitions(self) -> List[ResponderDefinition]: + return self._wrap(self._api.do_get('responderdefinition').json(), ResponderDefinition) + + def enable(self, responder_name, config) -> Responder: + url = 'organization/responder/{}'.format(responder_name) + config['name'] = responder_name + + return self._wrap(self._api.do_post(url, config).json(), Responder) + + def update(self, worker_id, config) -> Responder: + url = 'responder/{}'.format(worker_id) + config.pop('name', None) + + return self._wrap(self._api.do_patch(url, config).json(), Responder) + + def disable(self, worker_id) -> bool: + return self._api.do_delete('responder/{}'.format(worker_id)) + + def run_by_id(self, worker_id, data, **kwargs) -> Job: + tlp = data.get('tlp', 2) + data_type = data.get('dataType', None) + + post = { + 'dataType': data_type, + 'tlp': tlp + } + + # add additional details + for key in ['message', 'parameters']: + if key in data: + post[key] = data.get(key, None) + + post['data'] = data.get('data') + + return self._wrap(self._api.do_post('responder/{}/run'.format(worker_id), post).json(), Job) + + def run_by_name(self, responder_name, data, **kwargs) -> Job: + responder = self.get_by_name(responder_name) + + return self.run_by_id(responder.id, data, **kwargs) diff --git a/cortex4py/models/__init__.py b/cortex4py/models/__init__.py index ec586d6..771cdfc 100644 --- a/cortex4py/models/__init__.py +++ b/cortex4py/models/__init__.py @@ -2,5 +2,7 @@ from .user import User from .analyzer import Analyzer from .analyzer_definition import AnalyzerDefinition +from .responder import Responder +from .responder_definition import ResponderDefinition from .job import Job from .job_artifact import JobArtifact diff --git a/cortex4py/models/analyzer.py b/cortex4py/models/analyzer.py index 457f11e..110eb96 100644 --- a/cortex4py/models/analyzer.py +++ b/cortex4py/models/analyzer.py @@ -7,7 +7,7 @@ def __init__(self, data): defaults = { 'id': None, 'name': None, - 'analyzerDefinitionId': None, + 'workerDefinitionId': None, 'description': None, 'version': None, 'author': None, @@ -17,7 +17,9 @@ def __init__(self, data): 'configuration': {}, 'rate': None, 'rateUnit': None, - 'jobCache': None + 'jobCache': None, + 'maxPap': None, + 'maxTlp': None } if data is None: diff --git a/cortex4py/models/analyzer_definition.py b/cortex4py/models/analyzer_definition.py index 3462e5e..ecde237 100644 --- a/cortex4py/models/analyzer_definition.py +++ b/cortex4py/models/analyzer_definition.py @@ -7,7 +7,6 @@ def __init__(self, data): defaults = { 'id': None, 'name': None, - 'analyzerDefinitionId': None, 'description': None, 'version': None, 'author': None, diff --git a/cortex4py/models/job.py b/cortex4py/models/job.py index f6f2538..136990f 100644 --- a/cortex4py/models/job.py +++ b/cortex4py/models/job.py @@ -6,10 +6,11 @@ class Job(Model): def __init__(self, data): defaults = { 'id': None, + 'type': None, 'organization': None, - 'analyzerId': None, - 'analyzerDefinitionId': None, - 'analyzerName': None, + 'workerId': None, + 'workerDefinitionId': None, + 'workerName': None, 'status': None, 'dataType': None, 'tlp': 1, diff --git a/cortex4py/models/responder.py b/cortex4py/models/responder.py new file mode 100644 index 0000000..2f2e741 --- /dev/null +++ b/cortex4py/models/responder.py @@ -0,0 +1,25 @@ +from .model import Model + + +class Responder(Model): + + def __init__(self, data): + defaults = { + 'id': None, + 'name': None, + 'workerDefinitionId': None, + 'description': None, + 'version': None, + 'author': None, + 'url': None, + 'license': None, + 'dataTypeList': [], + 'configuration': {}, + 'rate': None, + 'rateUnit': None + } + + if data is None: + data = dict(defaults) + + self.__dict__ = {k: v for k, v in data.items() if not k.startswith('_')} \ No newline at end of file diff --git a/cortex4py/models/responder_definition.py b/cortex4py/models/responder_definition.py new file mode 100644 index 0000000..993b692 --- /dev/null +++ b/cortex4py/models/responder_definition.py @@ -0,0 +1,23 @@ +from .model import Model + + +class ResponderDefinition(Model): + + def __init__(self, data): + defaults = { + 'id': None, + 'name': None, + 'description': None, + 'version': None, + 'author': None, + 'url': None, + 'license': None, + 'basicConfig': None, + 'dataTypeList': [], + 'configurationItems': [] + } + + if data is None: + data = dict(defaults) + + self.__dict__ = {k: v for k, v in data.items() if not k.startswith('_')} \ No newline at end of file diff --git a/setup.py b/setup.py index 7e2c15e..57d00e7 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='cortex4py', - version='2.0.1', + version='2.1.0', description='Python API client for Cortex.', long_description=read_md('README.md'), author='TheHive-Project',