set default user-agent to flyscrape/0.1

2 files changed, 23 insertions, 1 deletions

diff --git a/fetch.go b/fetch.go
index f9d49d7..d969a74 100644
--- a/fetch.go
+++ b/fetch.go
@@ -13,6 +13,8 @@ import (
 	"github.com/cornelk/hashmap"
 )
 
+const userAgent = "flyscrape/0.1"
+
 func ProxiedFetch(proxyURL string) FetchFunc {
 	pu, err := url.Parse(proxyURL)
 	if err != nil {
@@ -63,7 +65,14 @@ func CachedFetch(fetch FetchFunc) FetchFunc {
 
 func Fetch() FetchFunc {
 	return func(url string) (string, error) {
-		resp, err := http.Get(url)
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return "", err
+		}
+
+		req.Header.Set("User-Agent", userAgent)
+
+		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
 			return "", err
 		}
diff --git a/fetch_test.go b/fetch_test.go
index 041ade2..b32ac0f 100644
--- a/fetch_test.go
+++ b/fetch_test.go
@@ -53,3 +53,16 @@ func TestFetchProxiedFetch(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, html, "foobar")
 }
+
+func TestFetchUserAgent(t *testing.T) {
+	var userAgent string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		userAgent = r.Header.Get("User-Agent")
+	}))
+
+	fetch := flyscrape.Fetch()
+
+	_, err := fetch(srv.URL)
+	require.NoError(t, err)
+	require.Equal(t, "flyscrape/0.1", userAgent)
+}