Respect multiple urls in domain- and urlfilter

1 files changed, 9 insertions, 4 deletions

diff --git a/modules/domainfilter/domainfilter.go b/modules/domainfilter/domainfilter.go
index e8691d3..ec95d68 100644
--- a/modules/domainfilter/domainfilter.go
+++ b/modules/domainfilter/domainfilter.go
@@ -15,6 +15,7 @@ func init() {
 
 type Module struct {
 	URL            string   `json:"url"`
+	URLs           []string `json:"urls"`
 	AllowedDomains []string `json:"allowedDomains"`
 	BlockedDomains []string `json:"blockedDomains"`
 
@@ -29,11 +30,15 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo {
 }
 
 func (m *Module) Provision(v flyscrape.Context) {
-	if m.URL == "" {
-		return
+	if m.URL != "" {
+		if u, err := url.Parse(m.URL); err == nil {
+			m.AllowedDomains = append(m.AllowedDomains, u.Host())
+		}
 	}
-	if u, err := url.Parse(m.URL); err == nil {
-		m.AllowedDomains = append(m.AllowedDomains, u.Host())
+	for _, u := range m.URLs {
+		if u, err := url.Parse(u); err == nil {
+			m.AllowedDomains = append(m.AllowedDomains, u.Host())
+		}
 	}
 }