diff options
Diffstat (limited to 'scrape.go')
| -rw-r--r-- | scrape.go | 9 |
1 files changed, 1 insertions, 8 deletions
@@ -134,7 +134,7 @@ func (s *Scraper) process(url string, depth int) { request := &Request{ Method: http.MethodGet, URL: url, - Headers: defaultHeaders(), + Headers: http.Header{}, Cookies: s.Client.Jar, Depth: depth, } @@ -231,10 +231,3 @@ func (s *Scraper) enqueueJob(url string, depth int) { s.wg.Done() } } - -func defaultHeaders() http.Header { - h := http.Header{} - h.Set("User-Agent", "flyscrape/0.1") - - return h -} |