Refactor codebase into modules

2 files changed, 31 insertions, 11 deletions

diff --git a/modules/starturl/starturl.go b/modules/starturl/starturl.go
index 109d28f..9e3ec31 100644
--- a/modules/starturl/starturl.go
+++ b/modules/starturl/starturl.go
@@ -9,15 +9,29 @@ import (
 )
 
 func init() {
-	flyscrape.RegisterModule(new(Module))
+	flyscrape.RegisterModule(Module{})
 }
 
 type Module struct {
 	URL string `json:"url"`
 }
 
-func (m *Module) OnLoad(v flyscrape.Visitor) {
-	v.Visit(m.URL)
+func (Module) ModuleInfo() flyscrape.ModuleInfo {
+	return flyscrape.ModuleInfo{
+		ID:  "starturl",
+		New: func() flyscrape.Module { return new(Module) },
+	}
 }
 
-var _ flyscrape.OnLoad = (*Module)(nil)
+func (m *Module) Provision(ctx flyscrape.Context) {
+	if m.disabled() {
+		return
+	}
+	ctx.Visit(m.URL)
+}
+
+func (m *Module) disabled() bool {
+	return m.URL == ""
+}
+
+var _ flyscrape.Provisioner = (*Module)(nil)
diff --git a/modules/starturl/starturl_test.go b/modules/starturl/starturl_test.go
index 6fab776..86e4ad7 100644
--- a/modules/starturl/starturl_test.go
+++ b/modules/starturl/starturl_test.go
@@ -5,23 +5,29 @@
 package starturl_test
 
 import (
+	"net/http"
 	"testing"
 
 	"github.com/philippta/flyscrape"
+	"github.com/philippta/flyscrape/modules/hook"
 	"github.com/philippta/flyscrape/modules/starturl"
 	"github.com/stretchr/testify/require"
 )
 
 func TestStartURL(t *testing.T) {
-	scraper := flyscrape.NewScraper()
-	scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/foo/bar"})
-	scraper.SetTransport(flyscrape.MockTransport(200, ""))
-
 	var url string
 	var depth int
-	scraper.OnRequest(func(req *flyscrape.Request) {
-		url = req.URL
-		depth = req.Depth
+
+	scraper := flyscrape.NewScraper()
+	scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/foo/bar"})
+	scraper.LoadModule(hook.Module{
+		AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper {
+			return flyscrape.MockTransport(200, "")
+		},
+		BuildRequestFn: func(r *flyscrape.Request) {
+			url = r.URL
+			depth = r.Depth
+		},
 	})
 
 	scraper.Run()