From 6cc19d5c412b8adc89092702d4bc21b416fc4fae Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Sat, 24 Feb 2024 13:59:29 +0100 Subject: Browser rendering --- modules/ratelimit/ratelimit.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'modules/ratelimit/ratelimit.go') diff --git a/modules/ratelimit/ratelimit.go b/modules/ratelimit/ratelimit.go index 152c6fd..f68f8e9 100644 --- a/modules/ratelimit/ratelimit.go +++ b/modules/ratelimit/ratelimit.go @@ -17,8 +17,9 @@ func init() { } type Module struct { - Rate int `json:"rate"` - Concurrency int `json:"concurrency"` + Rate int `json:"rate"` + Concurrency int `json:"concurrency"` + Browser bool `json:"browser"` ticker *time.Ticker ratelimit chan struct{} @@ -46,6 +47,10 @@ func (m *Module) Provision(v flyscrape.Context) { }() } + if m.browserEnabled() && !m.concurrencyEnabled() { + m.Concurrency = 1 + } + if m.concurrencyEnabled() { m.concurrency = make(chan struct{}, m.Concurrency) for i := 0; i < m.Concurrency; i++ { @@ -83,6 +88,10 @@ func (m *Module) concurrencyEnabled() bool { return m.Concurrency > 0 } +func (m *Module) browserEnabled() bool { + return m.Browser +} + var ( _ flyscrape.TransportAdapter = (*Module)(nil) _ flyscrape.Provisioner = (*Module)(nil) -- cgit v1.2.3