summaryrefslogtreecommitdiff
path: root/scrape.go
diff options
context:
space:
mode:
Diffstat (limited to 'scrape.go')
-rw-r--r--scrape.go6
1 files changed, 6 insertions, 0 deletions
diff --git a/scrape.go b/scrape.go
index 019849d..f09cba6 100644
--- a/scrape.go
+++ b/scrape.go
@@ -10,6 +10,7 @@ import (
"log"
"net/http"
"net/http/cookiejar"
+ "strings"
"sync"
"github.com/cornelk/hashmap"
@@ -210,6 +211,11 @@ func (s *Scraper) process(url string, depth int) {
}
func (s *Scraper) enqueueJob(url string, depth int) {
+ url = strings.TrimSpace(url)
+ if url == "" {
+ return
+ }
+
if _, ok := s.visited.Get(url); ok {
return
}