summaryrefslogtreecommitdiff
path: root/scrape.go
diff options
context:
space:
mode:
Diffstat (limited to 'scrape.go')
-rw-r--r--scrape.go9
1 files changed, 1 insertions, 8 deletions
diff --git a/scrape.go b/scrape.go
index 5b0f4e0..1667f42 100644
--- a/scrape.go
+++ b/scrape.go
@@ -134,7 +134,7 @@ func (s *Scraper) process(url string, depth int) {
request := &Request{
Method: http.MethodGet,
URL: url,
- Headers: defaultHeaders(),
+ Headers: http.Header{},
Cookies: s.Client.Jar,
Depth: depth,
}
@@ -231,10 +231,3 @@ func (s *Scraper) enqueueJob(url string, depth int) {
s.wg.Done()
}
}
-
-func defaultHeaders() http.Header {
- h := http.Header{}
- h.Set("User-Agent", "flyscrape/0.1")
-
- return h
-}