From 08df9258a532b653c243e077e82491dbe62ad854 Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Sat, 23 Sep 2023 17:41:57 +0200 Subject: refactor scraper into modules --- modules/jsonprinter/jsonprinter_test.go | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 modules/jsonprinter/jsonprinter_test.go (limited to 'modules/jsonprinter/jsonprinter_test.go') diff --git a/modules/jsonprinter/jsonprinter_test.go b/modules/jsonprinter/jsonprinter_test.go new file mode 100644 index 0000000..29cc438 --- /dev/null +++ b/modules/jsonprinter/jsonprinter_test.go @@ -0,0 +1,47 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package jsonprinter_test + +import ( + "net/http" + "testing" + + "github.com/philippta/flyscrape" + "github.com/philippta/flyscrape/modules/depth" + "github.com/philippta/flyscrape/modules/followlinks" + "github.com/philippta/flyscrape/modules/starturl" + "github.com/stretchr/testify/require" +) + +func TestDepth(t *testing.T) { + scraper := flyscrape.NewScraper() + scraper.LoadModule(&starturl.Module{URL: "http://www.example.com/"}) + scraper.LoadModule(&followlinks.Module{}) + scraper.LoadModule(&depth.Module{Depth: 2}) + + scraper.SetTransport(func(r *http.Request) (*http.Response, error) { + switch r.URL.String() { + case "http://www.example.com/": + return flyscrape.MockResponse(200, `Google`) + case "http://www.google.com/": + return flyscrape.MockResponse(200, `DuckDuckGo`) + case "http://www.duckduckgo.com/": + return flyscrape.MockResponse(200, `Example`) + } + return flyscrape.MockResponse(200, "") + }) + + var urls []string + scraper.OnRequest(func(req *flyscrape.Request) { + urls = append(urls, req.URL) + }) + + scraper.Run() + + require.Len(t, urls, 3) + require.Contains(t, urls, "http://www.example.com/") + require.Contains(t, urls, "http://www.google.com/") + require.Contains(t, urls, "http://www.duckduckgo.com/") +} -- cgit v1.2.3