From 94da9293f63e46712b0a890e1e0eab4153fdb3f9 Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Wed, 15 Nov 2023 16:31:50 +0100
Subject: Add file download functionality

---
 modules/cache/cache.go       | 13 ++++++++++++-
 modules/cache/sqlitestore.go |  2 +-
 2 files changed, 13 insertions(+), 2 deletions(-)

(limited to 'modules')

diff --git a/modules/cache/cache.go b/modules/cache/cache.go
index 4750e55..401aa49 100644
--- a/modules/cache/cache.go
+++ b/modules/cache/cache.go
@@ -51,8 +51,11 @@ func (m *Module) AdaptTransport(t http.RoundTripper) http.RoundTripper {
 		return t
 	}
 	return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
-		key := r.Method + " " + r.URL.String()
+		if nocache(r) {
+			return t.RoundTrip(r)
+		}
 
+		key := r.Method + " " + r.URL.String()
 		if b, ok := m.store.Get(key); ok {
 			if resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(b)), r); err == nil {
 				return resp, nil
@@ -86,6 +89,14 @@ func (m *Module) Finalize() {
 	}
 }
 
+func nocache(r *http.Request) bool {
+	if r.Header.Get(flyscrape.HeaderBypassCache) != "" {
+		r.Header.Del(flyscrape.HeaderBypassCache)
+		return true
+	}
+	return false
+}
+
 func replaceExt(filePath string, newExt string) string {
 	ext := filepath.Ext(filePath)
 	if ext != "" {
diff --git a/modules/cache/sqlitestore.go b/modules/cache/sqlitestore.go
index 50c8007..778699b 100644
--- a/modules/cache/sqlitestore.go
+++ b/modules/cache/sqlitestore.go
@@ -40,7 +40,7 @@ func (s *SQLiteStore) Get(key string) ([]byte, bool) {
 
 func (s *SQLiteStore) Set(key string, value []byte) {
 	if _, err := s.db.Exec(`INSERT INTO cache (key, value) VALUES (?, ?)`, key, value); err != nil {
-		log.Printf("cache: failed to insert cache key %q: %v\n", key, value)
+		log.Printf("cache: failed to insert cache key %q: %v\n", key, err)
 	}
 }
 
-- 
cgit v1.2.3