From 1fc497fbdc79a43c62ac2e8eaf4827752dbeef8e Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Thu, 5 Oct 2023 14:53:37 +0200 Subject: Refactor codebase into modules --- modules/starturl/starturl_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'modules/starturl/starturl_test.go') diff --git a/modules/starturl/starturl_test.go b/modules/starturl/starturl_test.go index 6fab776..86e4ad7 100644 --- a/modules/starturl/starturl_test.go +++ b/modules/starturl/starturl_test.go @@ -5,23 +5,29 @@ package starturl_test import ( + "net/http" "testing" "github.com/philippta/flyscrape" + "github.com/philippta/flyscrape/modules/hook" "github.com/philippta/flyscrape/modules/starturl" "github.com/stretchr/testify/require" ) func TestStartURL(t *testing.T) { - scraper := flyscrape.NewScraper() - scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/foo/bar"}) - scraper.SetTransport(flyscrape.MockTransport(200, "")) - var url string var depth int - scraper.OnRequest(func(req *flyscrape.Request) { - url = req.URL - depth = req.Depth + + scraper := flyscrape.NewScraper() + scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/foo/bar"}) + scraper.LoadModule(hook.Module{ + AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper { + return flyscrape.MockTransport(200, "") + }, + BuildRequestFn: func(r *flyscrape.Request) { + url = r.URL + depth = r.Depth + }, }) scraper.Run() -- cgit v1.2.3