1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
package main
import (
"flag"
"fmt"
"log"
"os"
"time"
"flyscrape"
)
type RunCommand struct{}
func (c *RunCommand) Run(args []string) error {
fs := flag.NewFlagSet("flyscrape-run", flag.ContinueOnError)
noPrettyPrint := fs.Bool("no-pretty-print", false, "no-pretty-print")
fs.Usage = c.Usage
if err := fs.Parse(args); err != nil {
return err
} else if fs.NArg() == 0 || fs.Arg(0) == "" {
return fmt.Errorf("script path required")
} else if fs.NArg() > 1 {
return fmt.Errorf("too many arguments")
}
script := fs.Arg(0)
src, err := os.ReadFile(script)
if err != nil {
return fmt.Errorf("failed to read script %q: %w", script, err)
}
opts, scrape, err := flyscrape.Compile(string(src))
if err != nil {
return fmt.Errorf("failed to compile script: %w", err)
}
svc := flyscrape.Scraper{
ScrapeOptions: opts,
ScrapeFunc: scrape,
}
count := 0
start := time.Now()
for result := range svc.Scrape() {
if count > 0 {
fmt.Println(",")
}
if count == 0 {
fmt.Println("[")
}
if *noPrettyPrint {
fmt.Print(flyscrape.Print(result, " "))
} else {
fmt.Print(flyscrape.PrettyPrint(result, " "))
}
count++
}
fmt.Println("\n]")
log.Printf("Scraped %d websites in %v\n", count, time.Since(start))
return nil
}
func (c *RunCommand) Usage() {
fmt.Println(`
The run command runs the scraping script.
Usage:
flyscrape run SCRIPT
Arguments:
-no-pretty-print
Disables pretty printing of scrape results.
Examples:
# Run the script.
$ flyscrape run example.js
# Run the script with 10 concurrent requests.
$ flyscrape run -concurrent 10 example.js
# Run the script with pretty printing disabled.
$ flyscrape run -no-pretty-print example.js
`[1:])
}
|