diff options
| -rw-r--r-- | examples/urls.txt | 3 | ||||
| -rw-r--r-- | examples/urls_from_file.js | 11 | ||||
| -rw-r--r-- | flyscrape.go | 36 | ||||
| -rw-r--r-- | js.go | 17 | ||||
| -rw-r--r-- | js_test.go | 8 |
5 files changed, 69 insertions, 6 deletions
diff --git a/examples/urls.txt b/examples/urls.txt new file mode 100644 index 0000000..0d4b500 --- /dev/null +++ b/examples/urls.txt @@ -0,0 +1,3 @@ +https://news.ycombinator.com/newest +https://news.ycombinator.com/ask +https://news.ycombinator.com/show diff --git a/examples/urls_from_file.js b/examples/urls_from_file.js new file mode 100644 index 0000000..0231032 --- /dev/null +++ b/examples/urls_from_file.js @@ -0,0 +1,11 @@ +import urls from "./urls.txt" + +export const config = { + urls: urls.split("\n").filter(Boolean) +}; + +export default function({ doc }) { + return { + title: doc.find("title").text().trim(), + }; +} diff --git a/flyscrape.go b/flyscrape.go index 797d4c7..d7aa8f8 100644 --- a/flyscrape.go +++ b/flyscrape.go @@ -28,11 +28,20 @@ func Run(file string, overrides map[string]any) error { imports, wait := NewJSLibrary(client) defer wait() + pop, err := pushDir(file) + if err != nil { + return err + } + exports, err := Compile(string(src), imports) if err != nil { return fmt.Errorf("failed to compile script: %w", err) } + if err := pop(); err != nil { + return err + } + cfg := exports.Config() cfg = updateCfgMultiple(cfg, overrides) @@ -63,12 +72,21 @@ func Dev(file string, overrides map[string]any) error { imports, wait := NewJSLibrary(client) defer wait() + pop, err := pushDir(file) + if err != nil { + return err + } + exports, err := Compile(s, imports) if err != nil { printCompileErr(file, err) return nil } + if err := pop(); err != nil { + return err + } + cfg := exports.Config() cfg = updateCfgMultiple(cfg, overrides) cfg = updateCfg(cfg, "depth", 0) @@ -147,3 +165,21 @@ func updateCfgMultiple(cfg Config, updates map[string]any) Config { return []byte(c) } + +func pushDir(file string) (func() error, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working directory: %w", err) + } + if err := os.Chdir(filepath.Dir(file)); err != nil { + return nil, fmt.Errorf("failed to change working directory: %w", err) + } + pop := func() error { + if err := os.Chdir(cwd); err != nil { + return fmt.Errorf("failed to change working directory: %w", err) + } + return nil + } + return pop, nil + +} @@ -72,9 +72,19 @@ func Compile(src string, imports Imports) (Exports, error) { } func build(src string) (string, error) { - res := api.Transform(src, api.TransformOptions{ + res := api.Build(api.BuildOptions{ + Loader: map[string]api.Loader{ + ".txt": api.LoaderText, + ".json": api.LoaderJSON, + }, + Bundle: true, + Stdin: &api.StdinOptions{ + Contents: src, + ResolveDir: ".", + }, Platform: api.PlatformNode, Format: api.FormatCommonJS, + External: []string{"flyscrape"}, }) var errs []error @@ -89,8 +99,11 @@ func build(src string) (string, error) { if len(res.Errors) > 0 { return "", errors.Join(errs...) } + if len(res.OutputFiles) == 0 { + return "", errors.New("no output generated") + } - return string(res.Code), nil + return string(res.OutputFiles[0].Contents), nil } func vm(src string, imports Imports) (Exports, error) { @@ -216,8 +216,8 @@ func TestJSConfig(t *testing.T) { func TestJSImports(t *testing.T) { js := ` - import A from "pkg-a" - import { bar } from "pkg-a/pkg-b" + import A from "flyscrape" + import { bar } from "flyscrape/foo" export const config = {} export default function() {} @@ -226,10 +226,10 @@ func TestJSImports(t *testing.T) { export const b = bar() ` imports := flyscrape.Imports{ - "pkg-a": map[string]any{ + "flyscrape": map[string]any{ "foo": 10, }, - "pkg-a/pkg-b": map[string]any{ + "flyscrape/foo": map[string]any{ "bar": func() string { return "baz" }, |