diff options
Diffstat (limited to 'modules/ratelimit/ratelimit_test.go')
| -rw-r--r-- | modules/ratelimit/ratelimit_test.go | 18 |
1 files changed, 11 insertions, 7 deletions
diff --git a/modules/ratelimit/ratelimit_test.go b/modules/ratelimit/ratelimit_test.go index 5e91f8f..ffd061c 100644 --- a/modules/ratelimit/ratelimit_test.go +++ b/modules/ratelimit/ratelimit_test.go @@ -5,33 +5,37 @@ package ratelimit_test import ( + "net/http" "testing" "time" "github.com/philippta/flyscrape" "github.com/philippta/flyscrape/modules/followlinks" + "github.com/philippta/flyscrape/modules/hook" "github.com/philippta/flyscrape/modules/ratelimit" "github.com/philippta/flyscrape/modules/starturl" "github.com/stretchr/testify/require" ) func TestRatelimit(t *testing.T) { + var times []time.Time + scraper := flyscrape.NewScraper() scraper.LoadModule(&starturl.Module{URL: "http://www.example.com"}) scraper.LoadModule(&followlinks.Module{}) scraper.LoadModule(&ratelimit.Module{ Rate: 100, }) - - scraper.SetTransport(flyscrape.MockTransport(200, `<a href="foo">foo</a>`)) - - var times []time.Time - scraper.OnRequest(func(req *flyscrape.Request) { - times = append(times, time.Now()) + scraper.LoadModule(hook.Module{ + AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper { + return flyscrape.MockTransport(200, `<a href="foo">foo</a>`) + }, + BuildRequestFn: func(r *flyscrape.Request) { + times = append(times, time.Now()) + }, }) start := time.Now() - scraper.Run() first := times[0].Add(-10 * time.Millisecond) |