From 40e02d5d28e59dbeb6134afdce12536c602e6aa5 Mon Sep 17 00:00:00 2001
From: Philipp Tanlak <philipp.tanlak@gmail.com>
Date: Fri, 10 Jan 2025 13:09:50 +0100
Subject: Implement manual following (#82)

---
 modules/followlinks/followlinks.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'modules/followlinks/followlinks.go')

diff --git a/modules/followlinks/followlinks.go b/modules/followlinks/followlinks.go
index c1448be..3ce2797 100644
--- a/modules/followlinks/followlinks.go
+++ b/modules/followlinks/followlinks.go
@@ -18,7 +18,7 @@ func init() {
 }
 
 type Module struct {
-	Follow []string `json:"follow"`
+	Follow *[]string `json:"follow"`
 }
 
 func (Module) ModuleInfo() flyscrape.ModuleInfo {
@@ -29,18 +29,26 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo {
 }
 
 func (m *Module) Provision(ctx flyscrape.Context) {
-	if len(m.Follow) == 0 {
-		m.Follow = []string{"a[href]"}
+	if m.Follow == nil {
+		m.Follow = &[]string{"a[href]"}
 	}
 }
 
 func (m *Module) ReceiveResponse(resp *flyscrape.Response) {
+	if m.Follow == nil {
+		return
+	}
+
 	for _, link := range m.parseLinks(string(resp.Body), resp.Request.URL) {
 		resp.Visit(link)
 	}
 }
 
 func (m *Module) parseLinks(html string, origin string) []string {
+	if m.Follow == nil {
+		return nil
+	}
+
 	var links []string
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
 	if err != nil {
@@ -54,7 +62,7 @@ func (m *Module) parseLinks(html string, origin string) []string {
 
 	uniqueLinks := make(map[string]bool)
 
-	for _, selector := range m.Follow {
+	for _, selector := range *m.Follow {
 		attr := parseSelectorAttr(selector)
 		doc.Find(selector).Each(func(i int, s *goquery.Selection) {
 			link, _ := s.Attr(attr)
-- 
cgit v1.2.3