diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/cache/cache.go | 13 | ||||
| -rw-r--r-- | modules/cache/sqlitestore.go | 2 |
2 files changed, 13 insertions, 2 deletions
diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 4750e55..401aa49 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -51,8 +51,11 @@ func (m *Module) AdaptTransport(t http.RoundTripper) http.RoundTripper { return t } return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) { - key := r.Method + " " + r.URL.String() + if nocache(r) { + return t.RoundTrip(r) + } + key := r.Method + " " + r.URL.String() if b, ok := m.store.Get(key); ok { if resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(b)), r); err == nil { return resp, nil @@ -86,6 +89,14 @@ func (m *Module) Finalize() { } } +func nocache(r *http.Request) bool { + if r.Header.Get(flyscrape.HeaderBypassCache) != "" { + r.Header.Del(flyscrape.HeaderBypassCache) + return true + } + return false +} + func replaceExt(filePath string, newExt string) string { ext := filepath.Ext(filePath) if ext != "" { diff --git a/modules/cache/sqlitestore.go b/modules/cache/sqlitestore.go index 50c8007..778699b 100644 --- a/modules/cache/sqlitestore.go +++ b/modules/cache/sqlitestore.go @@ -40,7 +40,7 @@ func (s *SQLiteStore) Get(key string) ([]byte, bool) { func (s *SQLiteStore) Set(key string, value []byte) { if _, err := s.db.Exec(`INSERT INTO cache (key, value) VALUES (?, ?)`, key, value); err != nil { - log.Printf("cache: failed to insert cache key %q: %v\n", key, value) + log.Printf("cache: failed to insert cache key %q: %v\n", key, err) } } |