From 807796ad35b48c58f61f6c058e12ec10078fd0e3 Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Wed, 16 Aug 2023 18:25:04 +0200
Subject: create cli

---
 fetch.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 fetch.go

(limited to 'fetch.go')

diff --git a/fetch.go b/fetch.go
new file mode 100644
index 0000000..68b10b6
--- /dev/null
+++ b/fetch.go
@@ -0,0 +1,51 @@
+package flyscrape
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/cornelk/hashmap"
+)
+
+func CachedFetch() FetchFunc {
+	cache := hashmap.New[string, string]()
+
+	return func(url string) (string, error) {
+		if html, ok := cache.Get(url); ok {
+			return html, nil
+		}
+
+		resp, err := http.Get(url)
+		if err != nil {
+			return "", err
+		}
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return "", err
+		}
+
+		html := string(body)
+		cache.Set(url, html)
+		return html, nil
+	}
+}
+
+func Fetch() FetchFunc {
+	return func(url string) (string, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return "", err
+		}
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return "", err
+		}
+
+		html := string(body)
+		return html, nil
+	}
+}
-- 
cgit v1.2.3