Mark non 2XX status code as errors

1 files changed, 5 insertions, 0 deletions

diff --git a/scrape.go b/scrape.go
index 3e38e46..bf0e1d6 100644
--- a/scrape.go
+++ b/scrape.go
@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"net/http/cookiejar"
+	"strconv"
 	"sync"
 
 	"github.com/cornelk/hashmap"
@@ -187,6 +188,10 @@ func (s *Scraper) process(url string, depth int) {
 	response.StatusCode = resp.StatusCode
 	response.Headers = resp.Header
 
+	if response.StatusCode < 200 || response.StatusCode >= 300 {
+		response.Error = strconv.Itoa(response.StatusCode) + " " + http.StatusText(response.StatusCode)
+	}
+
 	response.Body, err = io.ReadAll(resp.Body)
 	if err != nil {
 		response.Error = err