diff options
Diffstat (limited to 'modules/ratelimit')
| -rw-r--r-- | modules/ratelimit/ratelimit.go | 13 |
1 files changed, 11 insertions, 2 deletions
diff --git a/modules/ratelimit/ratelimit.go b/modules/ratelimit/ratelimit.go index 152c6fd..f68f8e9 100644 --- a/modules/ratelimit/ratelimit.go +++ b/modules/ratelimit/ratelimit.go @@ -17,8 +17,9 @@ func init() { } type Module struct { - Rate int `json:"rate"` - Concurrency int `json:"concurrency"` + Rate int `json:"rate"` + Concurrency int `json:"concurrency"` + Browser bool `json:"browser"` ticker *time.Ticker ratelimit chan struct{} @@ -46,6 +47,10 @@ func (m *Module) Provision(v flyscrape.Context) { }() } + if m.browserEnabled() && !m.concurrencyEnabled() { + m.Concurrency = 1 + } + if m.concurrencyEnabled() { m.concurrency = make(chan struct{}, m.Concurrency) for i := 0; i < m.Concurrency; i++ { @@ -83,6 +88,10 @@ func (m *Module) concurrencyEnabled() bool { return m.Concurrency > 0 } +func (m *Module) browserEnabled() bool { + return m.Browser +} + var ( _ flyscrape.TransportAdapter = (*Module)(nil) _ flyscrape.Provisioner = (*Module)(nil) |