Refactor to prepare for builtin JS functions

3 files changed, 8 insertions, 130 deletions

diff --git a/cmd/flyscrape/dev.go b/cmd/flyscrape/dev.go
index 9ddb3bf..84a436b 100644
--- a/cmd/flyscrape/dev.go
+++ b/cmd/flyscrape/dev.go
@@ -5,16 +5,9 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
-	"log"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
 
-	"github.com/inancgumus/screen"
 	"github.com/philippta/flyscrape"
 )
 
@@ -27,46 +20,13 @@ func (c *DevCommand) Run(args []string) error {
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
-		return fmt.Errorf("script path required")
+		c.Usage()
+		return flag.ErrHelp
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	script := fs.Arg(0)
-	cachefile, err := newCacheFile()
-	if err != nil {
-		return fmt.Errorf("failed to create cache file: %w", err)
-	}
-
-	trapsignal(func() { os.RemoveAll(cachefile) })
-
-	err = flyscrape.Watch(script, func(s string) error {
-		cfg, scrape, err := flyscrape.Compile(s)
-		if err != nil {
-			printCompileErr(script, err)
-			return nil
-		}
-
-		cfg = updateCfg(cfg, "depth", 0)
-		cfg = updateCfg(cfg, "cache", "file:"+cachefile)
-
-		scraper := flyscrape.NewScraper()
-		scraper.ScrapeFunc = scrape
-		scraper.Script = script
-
-		flyscrape.LoadModules(scraper, cfg)
-
-		screen.Clear()
-		screen.MoveTopLeft()
-		scraper.Run()
-
-		return nil
-	})
-	if err != nil && err != flyscrape.StopWatch {
-		return fmt.Errorf("failed to watch script %q: %w", script, err)
-	}
-
-	return nil
+	return flyscrape.Dev(fs.Arg(0))
 }
 
 func (c *DevCommand) Usage() {
@@ -78,58 +38,9 @@ Usage:
 
     flyscrape dev SCRIPT
 
-
 Examples:
 
     # Run and watch script.
     $ flyscrape dev example.js
 `[1:])
 }
-
-func printCompileErr(script string, err error) {
-	screen.Clear()
-	screen.MoveTopLeft()
-
-	if errs, ok := err.(interface{ Unwrap() []error }); ok {
-		for _, err := range errs.Unwrap() {
-			log.Printf("%s:%v\n", script, err)
-		}
-	} else {
-		log.Println(err)
-	}
-}
-
-func updateCfg(cfg flyscrape.Config, key string, value any) flyscrape.Config {
-	var m map[string]any
-	if err := json.Unmarshal(cfg, &m); err != nil {
-		return cfg
-	}
-
-	m[key] = value
-
-	b, err := json.Marshal(m)
-	if err != nil {
-		return cfg
-	}
-
-	return b
-}
-
-func newCacheFile() (string, error) {
-	cachedir, err := os.MkdirTemp("", "flyscrape-cache")
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(cachedir, "dev.cache"), nil
-}
-
-func trapsignal(f func()) {
-	sig := make(chan os.Signal, 2)
-	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
-
-	go func() {
-		<-sig
-		f()
-		os.Exit(0)
-	}()
-}
diff --git a/cmd/flyscrape/new.go b/cmd/flyscrape/new.go
index 8c0d6c4..4ab248e 100644
--- a/cmd/flyscrape/new.go
+++ b/cmd/flyscrape/new.go
@@ -21,7 +21,8 @@ func (c *NewCommand) Run(args []string) error {
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
-		return fmt.Errorf("script path required")
+		c.Usage()
+		return flag.ErrHelp
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
@@ -47,7 +48,6 @@ Usage:
 
     flyscrape new SCRIPT
 
-
 Examples:
 
     # Create a new scraping script.
diff --git a/cmd/flyscrape/run.go b/cmd/flyscrape/run.go
index 039574b..7a8930a 100644
--- a/cmd/flyscrape/run.go
+++ b/cmd/flyscrape/run.go
@@ -7,12 +7,8 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
-	"os"
-	"time"
 
 	"github.com/philippta/flyscrape"
-	"github.com/philippta/flyscrape/modules/hook"
 )
 
 type RunCommand struct{}
@@ -24,41 +20,13 @@ func (c *RunCommand) Run(args []string) error {
 	if err := fs.Parse(args); err != nil {
 		return err
 	} else if fs.NArg() == 0 || fs.Arg(0) == "" {
-		return fmt.Errorf("script path required")
+		c.Usage()
+		return flag.ErrHelp
 	} else if fs.NArg() > 1 {
 		return fmt.Errorf("too many arguments")
 	}
 
-	script := fs.Arg(0)
-	src, err := os.ReadFile(script)
-	if err != nil {
-		return fmt.Errorf("failed to read script %q: %w", script, err)
-	}
-
-	cfg, scrape, err := flyscrape.Compile(string(src))
-	if err != nil {
-		return fmt.Errorf("failed to compile script: %w", err)
-	}
-
-	scraper := flyscrape.NewScraper()
-	scraper.ScrapeFunc = scrape
-	scraper.Script = script
-
-	flyscrape.LoadModules(scraper, cfg)
-
-	count := 0
-	start := time.Now()
-
-	scraper.LoadModule(hook.Module{
-		ReceiveResponseFn: func(r *flyscrape.Response) {
-			count++
-		},
-	})
-
-	scraper.Run()
-
-	log.Printf("Scraped %d websites in %v\n", count, time.Since(start))
-	return nil
+	return flyscrape.Run(fs.Arg(0))
 }
 
 func (c *RunCommand) Usage() {
@@ -69,7 +37,6 @@ Usage:
 
     flyscrape run SCRIPT
 
-
 Examples:
 
     # Run the script.