From 80e24f0c780725bc854362def00211e88cb673bd Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Sat, 17 Feb 2024 21:49:05 +0100 Subject: Respect multiple urls in domain- and urlfilter --- modules/domainfilter/domainfilter.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'modules/domainfilter/domainfilter.go') diff --git a/modules/domainfilter/domainfilter.go b/modules/domainfilter/domainfilter.go index e8691d3..ec95d68 100644 --- a/modules/domainfilter/domainfilter.go +++ b/modules/domainfilter/domainfilter.go @@ -15,6 +15,7 @@ func init() { type Module struct { URL string `json:"url"` + URLs []string `json:"urls"` AllowedDomains []string `json:"allowedDomains"` BlockedDomains []string `json:"blockedDomains"` @@ -29,11 +30,15 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo { } func (m *Module) Provision(v flyscrape.Context) { - if m.URL == "" { - return + if m.URL != "" { + if u, err := url.Parse(m.URL); err == nil { + m.AllowedDomains = append(m.AllowedDomains, u.Host()) + } } - if u, err := url.Parse(m.URL); err == nil { - m.AllowedDomains = append(m.AllowedDomains, u.Host()) + for _, u := range m.URLs { + if u, err := url.Parse(u); err == nil { + m.AllowedDomains = append(m.AllowedDomains, u.Host()) + } } } -- cgit v1.2.3