From 92baa1671dd2a9dcc43d14f3a893f0e7f9a4b34d Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Mon, 26 Feb 2024 21:05:35 +0100
Subject: Improve cookie support for browser mode

---
 scrape.go | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

(limited to 'scrape.go')

diff --git a/scrape.go b/scrape.go
index cb7f18c..d12a5e0 100644
--- a/scrape.go
+++ b/scrape.go
@@ -123,7 +123,6 @@ func (s *Scraper) scrape() {
 	for i := 0; i < 500; i++ {
 		go func() {
 			for job := range s.jobs {
-				job := job
 				s.process(job.url, job.depth)
 				s.wg.Done()
 			}
@@ -197,11 +196,19 @@ func (s *Scraper) process(url string, depth int) {
 	}
 
 	if s.ScrapeFunc != nil {
-		response.Data, err = s.ScrapeFunc(ScrapeParams{HTML: string(response.Body), URL: request.URL})
-		if err != nil {
-			response.Error = err
-			return
-		}
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					log.Println(r)
+				}
+			}()
+
+			response.Data, err = s.ScrapeFunc(ScrapeParams{HTML: string(response.Body), URL: request.URL})
+			if err != nil {
+				response.Error = err
+				return
+			}
+		}()
 	}
 }
 
-- 
cgit v1.2.3