Merge pull request #487 from CIRCL/crawler_v2

fix: [crawler] error catcher
CIRCL · Apr 1, 2020 · 00573c9 · 00573c9
2 parents d72f28f + 179fba2
commit 00573c9
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/bin/torcrawler/TorSplashCrawler.py b/bin/torcrawler/TorSplashCrawler.py
@@ -156,7 +156,7 @@ def start_requests(self):
                 self.parse,
                 errback=self.errback_catcher,
                 endpoint='execute',
-                meta={'father': self.original_item},
+                meta={'father': self.original_item, 'current_url': self.start_urls},
                 args=l_cookies
             )
 
@@ -217,7 +217,7 @@ def parse(self,response):
                         self.parse,
                         errback=self.errback_catcher,
                         endpoint='execute',
-                        meta={'father': item_id},
+                        meta={'father': item_id, 'current_url': link.url},
                         args=l_cookies
                     )
 
@@ -227,7 +227,7 @@ def errback_catcher(self, failure):
 
             if failure.check(ResponseNeverReceived):
                 request = failure.request
-                url= response.data['last_url']
+                url= request.meta['current_url']
                 father = request.meta['father']
 
                 self.logger.error('Splash, ResponseNeverReceived for %s, retry in 10s ...', url)
@@ -242,7 +242,7 @@ def errback_catcher(self, failure):
                     errback=self.errback_catcher,
                     endpoint='execute',
                     cache_args=['lua_source'],
-                    meta={'father': father},
+                    meta={'father': father, 'current_url': url},
                     args=self.build_request_arg(response.cookiejar)
                 )