blob: 7c49dbfe5e9b6a724fe11da8ab640655668b52e9 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package cmd
import (
_ "embed"
"flag"
"fmt"
"log"
"os"
"strings"
_ "github.com/philippta/flyscrape/modules/cache"
_ "github.com/philippta/flyscrape/modules/depth"
_ "github.com/philippta/flyscrape/modules/domainfilter"
_ "github.com/philippta/flyscrape/modules/followlinks"
_ "github.com/philippta/flyscrape/modules/jsonprint"
_ "github.com/philippta/flyscrape/modules/proxy"
_ "github.com/philippta/flyscrape/modules/ratelimit"
_ "github.com/philippta/flyscrape/modules/starturl"
_ "github.com/philippta/flyscrape/modules/urlfilter"
)
func main() {
log.SetFlags(0)
m := &Main{}
if err := m.Run(os.Args[1:]); err == flag.ErrHelp {
os.Exit(1)
} else if err != nil {
log.Println(err)
os.Exit(1)
}
}
type Main struct{}
func (m *Main) Run(args []string) error {
var cmd string
if len(args) > 0 {
cmd, args = args[0], args[1:]
}
switch cmd {
case "new":
return (&NewCommand{}).Run(args)
case "run":
return (&RunCommand{}).Run(args)
case "dev":
return (&DevCommand{}).Run(args)
default:
if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
m.Usage()
return flag.ErrHelp
}
return fmt.Errorf("flyscrape %s: unknown command", cmd)
}
}
func (m *Main) Usage() {
fmt.Println(`
flyscrape is a standalone and scriptable web scraper for efficiently extracting data from websites.
Usage:
flyscrape <command> [arguments]
Commands:
new creates a sample scraping script
run runs a scraping script
dev watches and re-runs a scraping script
`[1:])
}
|