From c6950bcd5cd8fe9e7cc63fde7216a5a9b93b8aa0 Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Mon, 18 Sep 2023 21:42:06 +0200 Subject: rename watch command to dev --- cmd/flyscrape/dev.go | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ cmd/flyscrape/main.go | 6 +-- cmd/flyscrape/watch.go | 100 ------------------------------------------------- 3 files changed, 103 insertions(+), 103 deletions(-) create mode 100644 cmd/flyscrape/dev.go delete mode 100644 cmd/flyscrape/watch.go (limited to 'cmd') diff --git a/cmd/flyscrape/dev.go b/cmd/flyscrape/dev.go new file mode 100644 index 0000000..85ac1a1 --- /dev/null +++ b/cmd/flyscrape/dev.go @@ -0,0 +1,100 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package main + +import ( + "flag" + "fmt" + "log" + + "github.com/inancgumus/screen" + "github.com/philippta/flyscrape" +) + +type DevCommand struct{} + +func (c *DevCommand) Run(args []string) error { + fs := flag.NewFlagSet("flyscrape-dev", flag.ContinueOnError) + proxy := fs.String("proxy", "", "proxy") + fs.Usage = c.Usage + + if err := fs.Parse(args); err != nil { + return err + } else if fs.NArg() == 0 || fs.Arg(0) == "" { + return fmt.Errorf("script path required") + } else if fs.NArg() > 1 { + return fmt.Errorf("too many arguments") + } + + var fetch flyscrape.FetchFunc + if *proxy != "" { + fetch = flyscrape.ProxiedFetch(*proxy) + } else { + fetch = flyscrape.Fetch() + } + + fetch = flyscrape.CachedFetch(fetch) + script := fs.Arg(0) + + err := flyscrape.Watch(script, func(s string) error { + opts, scrape, err := flyscrape.Compile(s) + if err != nil { + screen.Clear() + screen.MoveTopLeft() + + if errs, ok := err.(interface{ Unwrap() []error }); ok { + for _, err := range errs.Unwrap() { + log.Printf("%s:%v\n", script, err) + } + } else { + log.Println(err) + } + + // ignore compilation errors + return nil + } + + opts.Depth = 0 + scr := flyscrape.Scraper{ + ScrapeOptions: opts, + ScrapeFunc: scrape, + FetchFunc: fetch, + } + + result := <-scr.Scrape() + screen.Clear() + screen.MoveTopLeft() + + if result.Error != nil { + log.Println(result.Error) + return nil + } + + fmt.Println(flyscrape.PrettyPrint(result, "")) + return nil + }) + if err != nil && err != flyscrape.StopWatch { + return fmt.Errorf("failed to watch script %q: %w", script, err) + } + + return nil +} + +func (c *DevCommand) Usage() { + fmt.Println(` +The dev command watches the scraping script and re-runs it on any change. +Recursive scraping is disabled in this mode, only the initial URL will be scraped. + +Usage: + + flyscrape dev SCRIPT + + +Examples: + + # Run and watch script. + $ flyscrape dev example.js +`[1:]) +} diff --git a/cmd/flyscrape/main.go b/cmd/flyscrape/main.go index 4470717..4e448bb 100644 --- a/cmd/flyscrape/main.go +++ b/cmd/flyscrape/main.go @@ -38,8 +38,8 @@ func (m *Main) Run(args []string) error { return (&NewCommand{}).Run(args) case "run": return (&RunCommand{}).Run(args) - case "watch": - return (&WatchCommand{}).Run(args) + case "dev": + return (&DevCommand{}).Run(args) default: if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") { m.Usage() @@ -61,6 +61,6 @@ Commands: new creates a sample scraping script run runs a scraping script - watch watches and re-runs a scraping script + dev watches and re-runs a scraping script `[1:]) } diff --git a/cmd/flyscrape/watch.go b/cmd/flyscrape/watch.go deleted file mode 100644 index b8e3c37..0000000 --- a/cmd/flyscrape/watch.go +++ /dev/null @@ -1,100 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package main - -import ( - "flag" - "fmt" - "log" - - "github.com/inancgumus/screen" - "github.com/philippta/flyscrape" -) - -type WatchCommand struct{} - -func (c *WatchCommand) Run(args []string) error { - fs := flag.NewFlagSet("flyscrape-watch", flag.ContinueOnError) - proxy := fs.String("proxy", "", "proxy") - fs.Usage = c.Usage - - if err := fs.Parse(args); err != nil { - return err - } else if fs.NArg() == 0 || fs.Arg(0) == "" { - return fmt.Errorf("script path required") - } else if fs.NArg() > 1 { - return fmt.Errorf("too many arguments") - } - - var fetch flyscrape.FetchFunc - if *proxy != "" { - fetch = flyscrape.ProxiedFetch(*proxy) - } else { - fetch = flyscrape.Fetch() - } - - fetch = flyscrape.CachedFetch(fetch) - script := fs.Arg(0) - - err := flyscrape.Watch(script, func(s string) error { - opts, scrape, err := flyscrape.Compile(s) - if err != nil { - screen.Clear() - screen.MoveTopLeft() - - if errs, ok := err.(interface{ Unwrap() []error }); ok { - for _, err := range errs.Unwrap() { - log.Printf("%s:%v\n", script, err) - } - } else { - log.Println(err) - } - - // ignore compilation errors - return nil - } - - opts.Depth = 0 - scr := flyscrape.Scraper{ - ScrapeOptions: opts, - ScrapeFunc: scrape, - FetchFunc: fetch, - } - - result := <-scr.Scrape() - screen.Clear() - screen.MoveTopLeft() - - if result.Error != nil { - log.Println(result.Error) - return nil - } - - fmt.Println(flyscrape.PrettyPrint(result, "")) - return nil - }) - if err != nil && err != flyscrape.StopWatch { - return fmt.Errorf("failed to watch script %q: %w", script, err) - } - - return nil -} - -func (c *WatchCommand) Usage() { - fmt.Println(` -The watch command watches the scraping script and re-runs it on any change. -Recursive scraping is disabled in this mode, only the initial URL will be scraped. - -Usage: - - flyscrape watch SCRIPT - - -Examples: - - # Run and watch script. - $ flyscrape watch example.js -`[1:]) -} -- cgit v1.2.3