From 08df9258a532b653c243e077e82491dbe62ad854 Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Sat, 23 Sep 2023 17:41:57 +0200 Subject: refactor scraper into modules --- modules/followlinks/followlinks.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 modules/followlinks/followlinks.go (limited to 'modules/followlinks/followlinks.go') diff --git a/modules/followlinks/followlinks.go b/modules/followlinks/followlinks.go new file mode 100644 index 0000000..dde0e90 --- /dev/null +++ b/modules/followlinks/followlinks.go @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package followlinks + +import ( + "github.com/philippta/flyscrape" +) + +func init() { + flyscrape.RegisterModule(new(Module)) +} + +type Module struct{} + +func (m *Module) ID() string { + return "followlinks" +} + +func (m *Module) OnResponse(resp *flyscrape.Response) { + for _, link := range flyscrape.ParseLinks(resp.HTML, resp.URL) { + resp.Visit(link) + } +} + +var ( + _ flyscrape.Module = (*Module)(nil) + _ flyscrape.OnResponse = (*Module)(nil) +) -- cgit v1.2.3