module.go


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

package flyscrape

import (
	"encoding/json"
	"net/http"
)

type Module any

type Transport interface {
	Transport(*http.Request) (*http.Response, error)
}

type CanRequest interface {
	CanRequest(url string, depth int) bool
}

type OnRequest interface {
	OnRequest(*Request)
}
type OnResponse interface {
	OnResponse(*Response)
}

type OnLoad interface {
	OnLoad(Visitor)
}

type OnComplete interface {
	OnComplete()
}

func RegisterModule(mod Module) {
	globalModules = append(globalModules, mod)
}

func LoadModules(s *Scraper, cfg Config) {
	for _, mod := range globalModules {
		json.Unmarshal(cfg, mod)
		s.LoadModule(mod)
	}
}

var globalModules = []Module{}