From 0b4c723d2c310310efba41f58ee3a626510bcd3a Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Sat, 17 Feb 2024 21:47:43 +0100
Subject: Mark non 2XX status code as errors

---
 scrape.go | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'scrape.go')

diff --git a/scrape.go b/scrape.go
index 3e38e46..bf0e1d6 100644
--- a/scrape.go
+++ b/scrape.go
@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"net/http/cookiejar"
+	"strconv"
 	"sync"
 
 	"github.com/cornelk/hashmap"
@@ -187,6 +188,10 @@ func (s *Scraper) process(url string, depth int) {
 	response.StatusCode = resp.StatusCode
 	response.Headers = resp.Header
 
+	if response.StatusCode < 200 || response.StatusCode >= 300 {
+		response.Error = strconv.Itoa(response.StatusCode) + " " + http.StatusText(response.StatusCode)
+	}
+
 	response.Body, err = io.ReadAll(resp.Body)
 	if err != nil {
 		response.Error = err
-- 
cgit v1.2.3