summaryrefslogtreecommitdiff
path: root/modules/depth
diff options
context:
space:
mode:
authorPhilipp Tanlak <philipp.tanlak@gmail.com>2023-09-24 23:36:00 +0200
committerPhilipp Tanlak <philipp.tanlak@gmail.com>2023-09-24 23:36:00 +0200
commitbd9e7f7acfd855d4685aa4544169c0e29cdbf205 (patch)
treec5218c65359c0c2dee5a8db2670f30db677f068a /modules/depth
parent08df9258a532b653c243e077e82491dbe62ad854 (diff)
clean up modules
Diffstat (limited to 'modules/depth')
-rw-r--r--modules/depth/depth.go9
-rw-r--r--modules/depth/depth_test.go14
2 files changed, 8 insertions, 15 deletions
diff --git a/modules/depth/depth.go b/modules/depth/depth.go
index 5efedc8..0cfbc71 100644
--- a/modules/depth/depth.go
+++ b/modules/depth/depth.go
@@ -16,15 +16,8 @@ type Module struct {
Depth int `json:"depth"`
}
-func (m *Module) ID() string {
- return "depth"
-}
-
func (m *Module) CanRequest(url string, depth int) bool {
return depth <= m.Depth
}
-var (
- _ flyscrape.Module = (*Module)(nil)
- _ flyscrape.CanRequest = (*Module)(nil)
-)
+var _ flyscrape.CanRequest = (*Module)(nil)
diff --git a/modules/depth/depth_test.go b/modules/depth/depth_test.go
index 309e628..c9afd6f 100644
--- a/modules/depth/depth_test.go
+++ b/modules/depth/depth_test.go
@@ -17,17 +17,17 @@ import (
func TestDepth(t *testing.T) {
scraper := flyscrape.NewScraper()
- scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/"})
+ scraper.LoadModule(&starturl.Module{URL: "http://www.example.com"})
scraper.LoadModule(&followlinks.Module{})
scraper.LoadModule(&depth.Module{Depth: 2})
scraper.SetTransport(func(r *http.Request) (*http.Response, error) {
switch r.URL.String() {
- case "http://www.example.com/":
+ case "http://www.example.com":
return flyscrape.MockResponse(200, `<a href="http://www.google.com">Google</a>`)
- case "http://www.google.com/":
+ case "http://www.google.com":
return flyscrape.MockResponse(200, `<a href="http://www.duckduckgo.com">DuckDuckGo</a>`)
- case "http://www.duckduckgo.com/":
+ case "http://www.duckduckgo.com":
return flyscrape.MockResponse(200, `<a href="http://www.example.com">Example</a>`)
}
return flyscrape.MockResponse(200, "")
@@ -41,7 +41,7 @@ func TestDepth(t *testing.T) {
scraper.Run()
require.Len(t, urls, 3)
- require.Contains(t, urls, "http://www.example.com/")
- require.Contains(t, urls, "http://www.google.com/")
- require.Contains(t, urls, "http://www.duckduckgo.com/")
+ require.Contains(t, urls, "http://www.example.com")
+ require.Contains(t, urls, "http://www.google.com")
+ require.Contains(t, urls, "http://www.duckduckgo.com")
}