path: root/module.go

package flyscrape

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
)

type Module interface {
	ID() string
}

type Transport interface {
	Transport(*http.Request) (*http.Response, error)
}

type CanRequest interface {
	CanRequest(url string, depth int) bool
}

type OnRequest interface {
	OnRequest(*Request)
}
type OnResponse interface {
	OnResponse(*Response)
}

type OnLoad interface {
	OnLoad(Visitor)
}

type OnComplete interface {
	OnComplete()
}

func RegisterModule(m Module) {
	id := m.ID()
	if id == "" {
		panic("module id is missing")
	}

	globalModulesMu.Lock()
	defer globalModulesMu.Unlock()

	if _, ok := globalModules[id]; ok {
		panic(fmt.Sprintf("module %s already registered", id))
	}
	globalModules[id] = m
}

func LoadModules(s *Scraper, opts Options) {
	globalModulesMu.RLock()
	defer globalModulesMu.RUnlock()

	for _, mod := range globalModules {
		json.Unmarshal(opts, mod)
		s.LoadModule(mod)
	}
}

var (
	globalModules   = map[string]Module{}
	globalModulesMu sync.RWMutex
)