From 8c68e0ed414bfb323d6e94db55c95db13797ef8e Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Mon, 4 Dec 2023 17:35:06 +0100 Subject: Make output file and format configurable (#24) --- cmd/flyscrape/main.go | 4 +++- cmd/main.go | 11 ----------- cmd/run.go | 6 ++++++ 3 files changed, 9 insertions(+), 12 deletions(-) (limited to 'cmd') diff --git a/cmd/flyscrape/main.go b/cmd/flyscrape/main.go index 8b9d430..9772648 100644 --- a/cmd/flyscrape/main.go +++ b/cmd/flyscrape/main.go @@ -15,7 +15,9 @@ import ( _ "github.com/philippta/flyscrape/modules/depth" _ "github.com/philippta/flyscrape/modules/domainfilter" _ "github.com/philippta/flyscrape/modules/followlinks" - _ "github.com/philippta/flyscrape/modules/jsonprint" + _ "github.com/philippta/flyscrape/modules/headers" + _ "github.com/philippta/flyscrape/modules/output/json" + _ "github.com/philippta/flyscrape/modules/output/ndjson" _ "github.com/philippta/flyscrape/modules/proxy" _ "github.com/philippta/flyscrape/modules/ratelimit" _ "github.com/philippta/flyscrape/modules/starturl" diff --git a/cmd/main.go b/cmd/main.go index eab3e03..b4f8d1d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -11,17 +11,6 @@ import ( "log" "os" "strings" - - _ "github.com/philippta/flyscrape/modules/cache" - _ "github.com/philippta/flyscrape/modules/depth" - _ "github.com/philippta/flyscrape/modules/domainfilter" - _ "github.com/philippta/flyscrape/modules/followlinks" - _ "github.com/philippta/flyscrape/modules/headers" - _ "github.com/philippta/flyscrape/modules/jsonprint" - _ "github.com/philippta/flyscrape/modules/proxy" - _ "github.com/philippta/flyscrape/modules/ratelimit" - _ "github.com/philippta/flyscrape/modules/starturl" - _ "github.com/philippta/flyscrape/modules/urlfilter" ) func main() { diff --git a/cmd/run.go b/cmd/run.go index 8c1a39d..a02b2ec 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -53,5 +53,11 @@ Examples: # Follow paginated links. $ flyscrape run example.js --depth 5 --follow ".next-button > a" + + # Set the output format to ndjson. + $ flyscrape run example.js --output.format ndjson + + # Write the output to a file. + $ flyscrape run example.js --output.file results.json `[1:]) } -- cgit v1.2.3