diff options
Diffstat (limited to 'modules/followlinks')
| -rw-r--r-- | modules/followlinks/followlinks.go | 11 | ||||
| -rw-r--r-- | modules/followlinks/followlinks_test.go | 2 |
2 files changed, 3 insertions, 10 deletions
diff --git a/modules/followlinks/followlinks.go b/modules/followlinks/followlinks.go index dde0e90..99d6cee 100644 --- a/modules/followlinks/followlinks.go +++ b/modules/followlinks/followlinks.go @@ -14,17 +14,10 @@ func init() { type Module struct{} -func (m *Module) ID() string { - return "followlinks" -} - func (m *Module) OnResponse(resp *flyscrape.Response) { - for _, link := range flyscrape.ParseLinks(resp.HTML, resp.URL) { + for _, link := range flyscrape.ParseLinks(string(resp.Body), resp.Request.URL) { resp.Visit(link) } } -var ( - _ flyscrape.Module = (*Module)(nil) - _ flyscrape.OnResponse = (*Module)(nil) -) +var _ flyscrape.OnResponse = (*Module)(nil) diff --git a/modules/followlinks/followlinks_test.go b/modules/followlinks/followlinks_test.go index 03c3a6b..18c8ceb 100644 --- a/modules/followlinks/followlinks_test.go +++ b/modules/followlinks/followlinks_test.go @@ -34,6 +34,6 @@ func TestFollowLinks(t *testing.T) { require.Contains(t, urls, "http://www.example.com/baz") require.Contains(t, urls, "http://www.example.com/foo/bar") require.Contains(t, urls, "http://www.example.com/foo/baz") - require.Contains(t, urls, "http://www.google.com/") + require.Contains(t, urls, "http://www.google.com") require.Contains(t, urls, "http://www.google.com/baz") } |