From 1fc497fbdc79a43c62ac2e8eaf4827752dbeef8e Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Thu, 5 Oct 2023 14:53:37 +0200
Subject: Refactor codebase into modules

---
 fetch.go | 89 ----------------------------------------------------------------
 1 file changed, 89 deletions(-)
 delete mode 100644 fetch.go

(limited to 'fetch.go')

diff --git a/fetch.go b/fetch.go
deleted file mode 100644
index d969a74..0000000
--- a/fetch.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package flyscrape
-
-import (
-	"crypto/tls"
-	"io"
-	"net/http"
-	"net/url"
-
-	"github.com/cornelk/hashmap"
-)
-
-const userAgent = "flyscrape/0.1"
-
-func ProxiedFetch(proxyURL string) FetchFunc {
-	pu, err := url.Parse(proxyURL)
-	if err != nil {
-		panic("invalid proxy url")
-	}
-
-	client := http.Client{
-		Transport: &http.Transport{
-			Proxy:           http.ProxyURL(pu),
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		},
-	}
-
-	return func(url string) (string, error) {
-		resp, err := client.Get(url)
-		if err != nil {
-			return "", err
-		}
-		defer resp.Body.Close()
-
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return "", err
-		}
-
-		html := string(body)
-		return html, nil
-	}
-}
-
-func CachedFetch(fetch FetchFunc) FetchFunc {
-	cache := hashmap.New[string, string]()
-
-	return func(url string) (string, error) {
-		if html, ok := cache.Get(url); ok {
-			return html, nil
-		}
-
-		html, err := fetch(url)
-		if err != nil {
-			return "", err
-		}
-
-		cache.Set(url, html)
-		return html, nil
-	}
-}
-
-func Fetch() FetchFunc {
-	return func(url string) (string, error) {
-		req, err := http.NewRequest(http.MethodGet, url, nil)
-		if err != nil {
-			return "", err
-		}
-
-		req.Header.Set("User-Agent", userAgent)
-
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return "", err
-		}
-		defer resp.Body.Close()
-
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return "", err
-		}
-
-		html := string(body)
-		return html, nil
-	}
-}
-- 
cgit v1.2.3