1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package proxy_test
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/philippta/flyscrape"
"github.com/philippta/flyscrape/modules/proxy"
"github.com/philippta/flyscrape/modules/starturl"
"github.com/stretchr/testify/require"
)
func TestProxy(t *testing.T) {
var called bool
p := newProxy(func() { called = true })
defer p.Close()
mods := []flyscrape.Module{
&starturl.Module{URL: "http://www.example.com"},
&proxy.Module{
Proxies: []string{p.URL},
},
}
scraper := flyscrape.NewScraper()
scraper.Modules = mods
scraper.Run()
require.True(t, called)
}
func TestProxyMultiple(t *testing.T) {
calls := []int{0, 0}
p0 := newProxy(func() { calls[0]++ })
p1 := newProxy(func() { calls[1]++ })
defer p0.Close()
defer p1.Close()
mod := &proxy.Module{Proxies: []string{p0.URL, p1.URL}}
mod.Provision(nil)
trans := mod.AdaptTransport(nil)
req := httptest.NewRequest("GET", "http://www.example.com/", nil)
for i := 0; i < 10; i++ {
resp, err := trans.RoundTrip(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
}
require.Greater(t, calls[0], 1)
require.Greater(t, calls[1], 1)
require.Equal(t, 10, calls[0]+calls[1])
}
func newProxy(f func()) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f()
w.Write([]byte("response from proxy"))
}))
}
|