diff options
| -rw-r--r-- | fetch.go | 11 | ||||
| -rw-r--r-- | fetch_test.go | 13 |
2 files changed, 23 insertions, 1 deletions
@@ -13,6 +13,8 @@ import ( "github.com/cornelk/hashmap" ) +const userAgent = "flyscrape/0.1" + func ProxiedFetch(proxyURL string) FetchFunc { pu, err := url.Parse(proxyURL) if err != nil { @@ -63,7 +65,14 @@ func CachedFetch(fetch FetchFunc) FetchFunc { func Fetch() FetchFunc { return func(url string) (string, error) { - resp, err := http.Get(url) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", userAgent) + + resp, err := http.DefaultClient.Do(req) if err != nil { return "", err } diff --git a/fetch_test.go b/fetch_test.go index 041ade2..b32ac0f 100644 --- a/fetch_test.go +++ b/fetch_test.go @@ -53,3 +53,16 @@ func TestFetchProxiedFetch(t *testing.T) { require.NoError(t, err) require.Equal(t, html, "foobar") } + +func TestFetchUserAgent(t *testing.T) { + var userAgent string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent = r.Header.Get("User-Agent") + })) + + fetch := flyscrape.Fetch() + + _, err := fetch(srv.URL) + require.NoError(t, err) + require.Equal(t, "flyscrape/0.1", userAgent) +} |