ensure FD close of logs

conf/default/integrations.conf.default

@@ -117,6 +117,20 @@ timeout = 60
 enabled = no
 timeout = 60
 
+# Community
+# FLARE capa -> to update rules: poetry run python utils/community.py -cr
+[flare_capa]
+enabled = no
+# Generate it always or generate on demand only(user need to click button to generate it), still should be enabled to use this feature on demand
+on_demand = no
+# Analyze binary payloads
+static = no
+# Analyze CAPE payloads
+cape = no
+# Analyze ProcDump
+procdump = no
+# behavior analysis summary
+behavior = no
 
 # external download services
 [downloaders]
@@ -127,3 +141,15 @@ order =
 [virustotal]
 enabled = no
 apikey =
+
+# Community
+[floss]
+enabled = no
+on_demand = yes
+static_strings = no
+stack_strings = yes
+decoded_strings = yes
+tight_strings = yes
+min_length = 5
+# Download FLOSS signatures from https://github.com/mandiant/flare-floss/tree/master/sigs
+sigs_path = data/flare-signatures

conf/default/processing.conf.default

@@ -22,22 +22,6 @@ enabled = no
 [analysisinfo]
 enabled = yes
 
-# Community
-# FLARE capa -> to update rules utils/community.py -cr
-# install -> cd /tmp && git clone --recurse-submodules https://github.com/fireeye/capa.git && cd capa && git submodule update --init rules && python -m poetry run pip install .
-[flare_capa]
-enabled = no
-# Generate it always or generate on demand only(user need to click button to generate it), still should be enabled to use this feature on demand
-on_demand = no
-# Analyze binary payloads
-static = no
-# Analyze CAPE payloads
-cape = no
-# Analyze ProcDump
-procdump = no
-# behavior analysis summary
-behavior = no
-
 # Community
 [decompression]
 enabled = no
@@ -328,17 +312,5 @@ key =
 [script_log_processing]
 enabled = yes
 
-# Community
-[floss]
-enabled = no
-on_demand = yes
-static_strings = no
-stack_strings = yes
-decoded_strings = yes
-tight_strings = yes
-min_length = 5
-# Download FLOSS signatures from https://github.com/mandiant/flare-floss/tree/master/sigs
-sigs_path = data/flare-signatures
-
 [html_scraper]
 enabled = no

conf/default/reporting.conf.default

@@ -4,11 +4,6 @@
 # You can also add additional options under the section of your module and
 # they will be available in your Python class.
 
-# Generate CAPE's analysis summary by FLARE/Mandiant's CAPA
-[flare_capa_summary]
-enabled = yes
-on_demand= no
-
 # Community
 [cents]
 enabled = no
@@ -223,4 +218,4 @@ enabled = no
 enabled = no
 
 [browserext]
-enabled = no
+enabled = no

lib/cuckoo/common/integrations/capa.py

@@ -17,13 +17,6 @@
 processing_conf = Config("processing")
 reporting_conf = Config("reporting")
 
-
-"""
-from lib.cuckoo.common.integrations.capa import flare_capa_details, HAVE_FLARE_CAPA
-path = "storage/binaries/8c4111e5ec6ec033ea32e7d40f3c36e16ad50146240dacfc3de6cf8df19e6531"
-details = flare_capa_details(path, "static", on_demand=True)
-"""
-
 rules = False
 HAVE_FLARE_CAPA = False
 if processing_conf.flare_capa.enabled or reporting_conf.flare_capa_summary.enabled:
@@ -261,6 +254,7 @@ def flare_capa_details(
         and not processing_conf.flare_capa.on_demand
         or on_demand
     ):
+        # ToDo check if PE file in TYPE
         try:
             file_path_object = path_object(file_path)
             # extract features and find capabilities
@@ -272,7 +266,7 @@ def flare_capa_details(
                 try:
                     extractor = capa.features.extractors.cape.extractor.CapeExtractor.from_report(results)
                 except ValidationError as e:
-                    log.exception("CAPA ValidationError %s", e)
+                    log.debug("CAPA ValidationError %s", e)
                     return {}
             else:
                 log.error("CAPA: Missed results probably")
@@ -300,3 +294,9 @@ def flare_capa_details(
             log.exception(e)
 
     return capa_output
+
+
+if __name__ == "__main__":
+    import sys
+    from lib.cuckoo.common.integrations.capa import flare_capa_details, HAVE_FLARE_CAPA
+    details = flare_capa_details(sys.argv[1], "static", on_demand=True)

lib/cuckoo/common/integrations/file_extra_info.py

@@ -74,11 +74,11 @@
 
 HAVE_FLARE_CAPA = False
 # required to not load not enabled dependencies
-if processing_conf.flare_capa.enabled and not processing_conf.flare_capa.on_demand:
+if integration_conf.flare_capa.enabled and not integration_conf.flare_capa.on_demand:
     from lib.cuckoo.common.integrations.capa import HAVE_FLARE_CAPA, flare_capa_details
 
 HAVE_FLOSS = False
-if processing_conf.floss.enabled and not processing_conf.floss.on_demand:
+if integration_conf.floss.enabled and not integration_conf.floss.on_demand:
     from lib.cuckoo.common.integrations.floss import HAVE_FLOSS, Floss
 
 log = logging.getLogger(__name__)
@@ -177,11 +177,12 @@ def static_file_info(
         data_dictionary["pe"] = PortableExecutable(file_path).run(task_id)
 
         if HAVE_FLARE_CAPA:
+            # https://github.com/mandiant/capa/issues/2620
             capa_details = flare_capa_details(file_path, "static")
             if capa_details:
                 data_dictionary["flare_capa"] = capa_details
 
-        if HAVE_FLOSS and processing_conf.floss.enabled and "Mono" not in data_dictionary["type"]:
+        if HAVE_FLOSS and integration_conf.floss.enabled and "Mono" not in data_dictionary["type"]:
             floss_strings = Floss(file_path, "static", "pe").run()
             if floss_strings:
                 data_dictionary["floss"] = floss_strings

lib/cuckoo/core/resultserver.py

@@ -172,6 +172,10 @@ def discard(self):
         while _ := self.read():
             pass
 
+    def __del__(self):
+        if self.sock:
+            self.sock.close()
+
 
 class WriteLimiter:
     def __init__(self, fd, remain):
@@ -197,6 +201,10 @@ def write(self, buf):
     def flush(self):
         self.fd.flush()
 
+    def __del__(self):
+        if self.fd:
+            self.fd.close()
+
 
 class FileUpload(ProtocolHandler):
     def init(self):
@@ -205,6 +213,10 @@ def init(self):
         self.fd = None
         self.filelog = os.path.join(self.handler.storagepath, "files.json")
 
+    def __del__(self):
+        if self.fd:
+            self.fd.close()
+
     def handle(self):
         # Read until newline for file path, e.g.,
         # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
@@ -299,6 +311,9 @@ def handle(self):
         if self.fd:
             return self.handler.copy_to_fd(self.fd)
 
+    def __del__(self):
+        if self.fd:
+            self.fd.close()
 
 TYPECONVERTERS = {"h": lambda v: f"0x{default_converter(v):08x}", "p": lambda v: f"0x{default_converter(v):08x}"}
 
@@ -405,6 +420,10 @@ def handle(self):
             self.handler.sock.settimeout(None)
             return self.handler.copy_to_fd(self.fd)
 
+    def __del__(self):
+        if self.fd:
+            self.fd.close()
+
 
 class GeventResultServerWorker(gevent.server.StreamServer):
     """The new ResultServer, providing a huge performance boost as well as
@@ -434,6 +453,9 @@ def __init__(self, *args, **kwargs):
 
         # Store running handlers for task_id
         self.handlers = {}
+        self.log_start_count = 0
+        self.log_stop_count = 0
+        self.task_id = None
 
     def do_run(self):
         self.serve_forever()
@@ -457,6 +479,10 @@ def del_task(self, task_id, ipaddr):
             for ctx in ctxs:
                 log.debug("Task #%s: Cancel %s", task_id, ctx)
                 ctx.cancel()
+            # ToDo just reinforce cleanup
+            task_log_stop(task_id)
+
+        task_log_stop(task_id, debug=True)
 
     def create_folders(self):
         for folder in list(RESULT_UPLOADABLE) + [b"logs"]:
@@ -480,17 +506,21 @@ def handle(self, sock, addr):
                 log.warning("ResultServer did not have a task for IP %s", ipaddr)
                 return
 
+        self.task_id = task_id
         self.storagepath = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id))
 
         # Create all missing folders for this analysis.
         self.create_folders()
 
         ctx = HandlerContext(task_id, self.storagepath, sock)
         task_log_start(task_id)
+        self.log_start_count += 1
         try:
             try:
+                log.debug("Task #%s: Negotiation started", task_id)
                 protocol = self.negotiate_protocol(task_id, ctx)
             except EOFError:
+                log.debug("Task #%s: Negotiation failed on start", task_id)
                 return
 
             # Registering the context allows us to abort the handler by
@@ -502,13 +532,16 @@ def handle(self, sock, addr):
                 # been registered
                 if self.tasks.get(ipaddr) != task_id:
                     log.warning("Task #%s for IP %s was cancelled during negotiation", task_id, ipaddr)
+                    log.debug("Task #%s: Negotiation failed inside task_mgmt_lock manager", task_id)
                     return
                 s = self.handlers.setdefault(task_id, set())
                 s.add(ctx)
 
             try:
                 with protocol:
+                    log.debug("Task #%s: Negotiation succeeded", task_id)
                     protocol.handle()
+                    log.debug("Task #%s: Negotiation succeeded handled", task_id)
             except CuckooOperationalError as e:
                 log.exception(e)
             finally:
@@ -519,7 +552,11 @@ def handle(self, sock, addr):
                     # This is usually not a good sign
                     log.warning("Task #%s with protocol %s has unprocessed data before getting disconnected", task_id, protocol)
         finally:
+            log.debug("Task #%s: Negotiation finished", task_id)
             task_log_stop(task_id)
+            self.log_stop_count += 1
+
+        log.debug("Task #%s: Connection closed. Start count %d, stop count %d", task_id, self.log_start_count, self.log_stop_count)
 
     def negotiate_protocol(self, task_id, ctx):
         header = ctx.read_newline()

modules/processing/CAPE.py

@@ -37,11 +37,12 @@
 )
 
 processing_conf = Config("processing")
+integrations_conf = Config("integrations")
 externalservices_conf = Config("externalservices")
 
 HAVE_FLARE_CAPA = False
 # required to not load not enabled dependencies
-if processing_conf.flare_capa.enabled and not processing_conf.flare_capa.on_demand:
+if integrations_conf.flare_capa.enabled and not integrations_conf.flare_capa.on_demand:
     from lib.cuckoo.common.integrations.capa import HAVE_FLARE_CAPA, flare_capa_details
 
 MISP_HASH_LOOKUP = False
@@ -245,7 +246,8 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
             if file_info.get("pid"):
                 _ = cape_name_from_yara(file_info, file_info["pid"], self.results)
 
-            if HAVE_FLARE_CAPA:
+            # ToDo https://github.com/mandiant/capa/issues/2620
+            if HAVE_FLARE_CAPA and ("PE32" in file_info["type"] or "MS-DOS executable" in file_info["type"]):
                 pretime = timeit.default_timer()
                 capa_details = flare_capa_details(file_path, "procdump")
                 if capa_details:
@@ -322,7 +324,8 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
                 append_file = is_duplicated_binary(file_info, cape_file, append_file)
 
         if append_file:
-            if HAVE_FLARE_CAPA and category == "CAPE":
+            # ToDo https://github.com/mandiant/capa/issues/2620
+            if HAVE_FLARE_CAPA and category == "CAPE" and ("PE32" in file_info["type"] or "MS-DOS executable" in file_info["type"]):
                 pretime = timeit.default_timer()
                 capa_details = flare_capa_details(file_path, "cape")
                 if capa_details:

modules/processing/behavior.py

@@ -26,11 +26,11 @@
 
 log = logging.getLogger(__name__)
 cfg = Config()
-processing_conf = Config("processing")
+integrations_conf = Config("integrations")
 
 HAVE_FLARE_CAPA = False
 # required to not load not enabled dependencies
-if processing_conf.flare_capa.enabled and processing_conf.flare_capa.behavior:
+if integrations_conf.flare_capa.enabled and integrations_conf.flare_capa.behavior:
     from lib.cuckoo.common.integrations.capa import HAVE_FLARE_CAPA, flare_capa_details
 
 
@@ -1234,6 +1234,10 @@ def run(self):
                 except Exception as e:
                     log.error("Behavior. Can't load json: %s", str(e))
 
-        if HAVE_FLARE_CAPA and self.results.get("info", {}).get("category", "") == "file":
-            behavior["capa"] = flare_capa_details(file_path=self.results["target"]["file"]["path"], category="behavior", backend="cape", results={"behavior": behavior, **self.results})
+        # https://github.com/mandiant/capa/issues/2620
+        if HAVE_FLARE_CAPA and self.results.get("info", {}).get("category", "") == "file" and "PE" in self.results.get("target", {}).get("file", "").get("type",""):
+            try:
+                self.results["capa_summary"] = flare_capa_details(file_path=self.results["target"]["file"]["path"], category="behavior", backend="cape", results={"behavior": behavior, **self.results})
+            except Exception as e:
+                log.error("Can't generate CAPA summary: %s", str(e))
         return behavior