From 49bde10c0b4d1c296960fbe94e30672da42df0e5 Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Mon, 18 Sep 2023 21:22:57 +0200
Subject: set default user-agent to flyscrape/0.1

---
 fetch.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'fetch.go')

diff --git a/fetch.go b/fetch.go
index f9d49d7..d969a74 100644
--- a/fetch.go
+++ b/fetch.go
@@ -13,6 +13,8 @@ import (
 	"github.com/cornelk/hashmap"
 )
 
+const userAgent = "flyscrape/0.1"
+
 func ProxiedFetch(proxyURL string) FetchFunc {
 	pu, err := url.Parse(proxyURL)
 	if err != nil {
@@ -63,7 +65,14 @@ func CachedFetch(fetch FetchFunc) FetchFunc {
 
 func Fetch() FetchFunc {
 	return func(url string) (string, error) {
-		resp, err := http.Get(url)
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return "", err
+		}
+
+		req.Header.Set("User-Agent", userAgent)
+
+		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
 			return "", err
 		}
-- 
cgit v1.2.3