From 5c16435e2218344a6e232ebb48cf022a32ba85d5 Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Sun, 27 Aug 2023 19:10:49 +0200
Subject: add tests and allow urls

---
 fetch.go | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)

(limited to 'fetch.go')

diff --git a/fetch.go b/fetch.go
index 8303a76..f9d49d7 100644
--- a/fetch.go
+++ b/fetch.go
@@ -5,21 +5,29 @@
 package flyscrape
 
 import (
+	"crypto/tls"
 	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/cornelk/hashmap"
 )
 
-func CachedFetch() FetchFunc {
-	cache := hashmap.New[string, string]()
+func ProxiedFetch(proxyURL string) FetchFunc {
+	pu, err := url.Parse(proxyURL)
+	if err != nil {
+		panic("invalid proxy url")
+	}
 
-	return func(url string) (string, error) {
-		if html, ok := cache.Get(url); ok {
-			return html, nil
-		}
+	client := http.Client{
+		Transport: &http.Transport{
+			Proxy:           http.ProxyURL(pu),
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
 
-		resp, err := http.Get(url)
+	return func(url string) (string, error) {
+		resp, err := client.Get(url)
 		if err != nil {
 			return "", err
 		}
@@ -31,6 +39,23 @@ func CachedFetch() FetchFunc {
 		}
 
 		html := string(body)
+		return html, nil
+	}
+}
+
+func CachedFetch(fetch FetchFunc) FetchFunc {
+	cache := hashmap.New[string, string]()
+
+	return func(url string) (string, error) {
+		if html, ok := cache.Get(url); ok {
+			return html, nil
+		}
+
+		html, err := fetch(url)
+		if err != nil {
+			return "", err
+		}
+
 		cache.Set(url, html)
 		return html, nil
 	}
-- 
cgit v1.2.3