// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. package flyscrape import ( "fmt" "log" "net/http" "os" "os/signal" "path/filepath" "syscall" "github.com/inancgumus/screen" "github.com/tidwall/sjson" ) var Version string func Run(file string, overrides map[string]any) error { src, err := os.ReadFile(file) if err != nil { return fmt.Errorf("failed to read script %q: %w", file, err) } client := &http.Client{} imports, wait := NewJSLibrary(client) defer wait() pop, err := pushDir(file) if err != nil { return err } exports, err := Compile(string(src), imports) if err != nil { return fmt.Errorf("failed to compile script: %w", err) } if err := pop(); err != nil { return err } cfg := exports.Config() cfg = updateCfgMultiple(cfg, overrides) scraper := NewScraper() scraper.ScrapeFunc = exports.Scrape scraper.Script = file scraper.Client = client scraper.Modules = LoadModules(cfg) scraper.Run() return nil } func Dev(file string, overrides map[string]any) error { cachefile, err := newCacheFile() if err != nil { return fmt.Errorf("failed to create cache file: %w", err) } trapsignal(func() { os.RemoveAll(cachefile) }) fn := func(s string) error { client := &http.Client{} imports, wait := NewJSLibrary(client) defer wait() pop, err := pushDir(file) if err != nil { return err } exports, err := Compile(s, imports) if err != nil { printCompileErr(file, err) return nil } if err := pop(); err != nil { return err } cfg := exports.Config() cfg = updateCfgMultiple(cfg, overrides) cfg = updateCfg(cfg, "depth", 0) cfg = updateCfg(cfg, "cache", "file:"+cachefile) scraper := NewScraper() scraper.ScrapeFunc = exports.Scrape scraper.Script = file scraper.Client = client scraper.Modules = LoadModules(cfg) screen.Clear() screen.MoveTopLeft() scraper.Run() return nil } if err := Watch(file, fn); err != nil && err != StopWatch { return fmt.Errorf("failed to watch script %q: %w", file, err) } return nil } func printCompileErr(script string, err error) { screen.Clear() screen.MoveTopLeft() if errs, ok := err.(interface{ Unwrap() []error }); ok { for _, err := range errs.Unwrap() { log.Printf("%s:%v\n", script, err) } } else { log.Println(err) } } func updateCfg(cfg Config, key string, value any) Config { newcfg, err := sjson.Set(string(cfg), key, value) if err != nil { return cfg } return Config(newcfg) } func newCacheFile() (string, error) { cachedir, err := os.MkdirTemp("", "flyscrape-cache") if err != nil { return "", err } return filepath.Join(cachedir, "dev.cache"), nil } func trapsignal(f func()) { sig := make(chan os.Signal, 2) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) go func() { <-sig f() os.Exit(0) }() } func updateCfgMultiple(cfg Config, updates map[string]any) Config { c := string(cfg) for k, v := range updates { nc, err := sjson.Set(c, k, v) if err != nil { continue } c = nc } return []byte(c) } func pushDir(file string) (func() error, error) { cwd, err := os.Getwd() if err != nil { return nil, fmt.Errorf("failed to get current working directory: %w", err) } if err := os.Chdir(filepath.Dir(file)); err != nil { return nil, fmt.Errorf("failed to change working directory: %w", err) } pop := func() error { if err := os.Chdir(cwd); err != nil { return fmt.Errorf("failed to change working directory: %w", err) } return nil } return pop, nil }