Generate random user agent when non is provided (#76)

1 files changed, 1 insertions, 8 deletions

diff --git a/scrape.go b/scrape.go
index 5b0f4e0..1667f42 100644
--- a/scrape.go
+++ b/scrape.go
@@ -134,7 +134,7 @@ func (s *Scraper) process(url string, depth int) {
 	request := &Request{
 		Method:  http.MethodGet,
 		URL:     url,
-		Headers: defaultHeaders(),
+		Headers: http.Header{},
 		Cookies: s.Client.Jar,
 		Depth:   depth,
 	}
@@ -231,10 +231,3 @@ func (s *Scraper) enqueueJob(url string, depth int) {
 		s.wg.Done()
 	}
 }
-
-func defaultHeaders() http.Header {
-	h := http.Header{}
-	h.Set("User-Agent", "flyscrape/0.1")
-
-	return h
-}