package js import ( "encoding/json" "errors" "fmt" "math/rand" "os" "strings" "time" "flyscrape/flyscrape" "flyscrape/js/jsbundle" "github.com/evanw/esbuild/pkg/api" v8 "rogchap.com/v8go" ) func init() { rand.Seed(time.Now().UnixNano()) } func Compile(file string) (*flyscrape.ScrapeOptions, flyscrape.ScrapeFunc, error) { src, err := build(file) if err != nil { return nil, nil, err } return vm(src) } func build(file string) (string, error) { data, err := os.ReadFile(file) if err != nil { return "", fmt.Errorf("read %q: %w", file, err) } res := api.Transform(string(data), api.TransformOptions{ Platform: api.PlatformBrowser, Format: api.FormatIIFE, }) var errs []error for _, msg := range res.Errors { errs = append(errs, fmt.Errorf("%s", msg.Text)) } if len(res.Errors) > 0 { return "", errors.Join(errs...) } return string(res.Code), nil } func vm(src string) (*flyscrape.ScrapeOptions, flyscrape.ScrapeFunc, error) { ctx := v8.NewContext() ctx.RunScript("var module = {}", "main.js") if _, err := ctx.RunScript(removeIIFE(jsbundle.Flyscrape), "main.js"); err != nil { return nil, nil, fmt.Errorf("run bundled js: %w", err) } if _, err := ctx.RunScript(`const require = () => require_flyscrape();`, "main.js"); err != nil { return nil, nil, fmt.Errorf("define require: %w", err) } if _, err := ctx.RunScript(removeIIFE(src), "main.js"); err != nil { return nil, nil, fmt.Errorf("userscript: %w", err) } var opts flyscrape.ScrapeOptions url, err := ctx.RunScript("options.url", "main.js") if err != nil { return nil, nil, fmt.Errorf("eval: options.url: %w", err) } opts.URL = url.String() depth, err := ctx.RunScript("options.depth", "main.js") if err != nil { return nil, nil, fmt.Errorf("eval: options.depth: %w", err) } opts.Depth = int(depth.Integer()) scrape := func(params flyscrape.ScrapeParams) (flyscrape.M, error) { suffix := randSeq(10) ctx.Global().Set("html_"+suffix, params.HTML) data, err := ctx.RunScript(fmt.Sprintf(`JSON.stringify(stdin_default({html: html_%s}))`, suffix), "main.js") if err != nil { return nil, err } var obj flyscrape.M if err := json.Unmarshal([]byte(data.String()), &obj); err != nil { return nil, err } return obj, nil } return &opts, scrape, nil } func randSeq(n int) string { letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") b := make([]rune, n) for i := range b { b[i] = letters[rand.Intn(len(letters))] } return string(b) } func removeIIFE(s string) string { s = strings.TrimPrefix(s, "(() => {\n") s = strings.TrimSuffix(s, "})();\n") return s }