path: root/modules/proxy/proxy_test.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package proxy_test

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/philippta/flyscrape"
	"github.com/philippta/flyscrape/modules/proxy"
	"github.com/philippta/flyscrape/modules/starturl"
	"github.com/stretchr/testify/require"
)

func TestProxy(t *testing.T) {
	var called bool
	p := newProxy(func() { called = true })
	defer p.Close()

	mods := []flyscrape.Module{
		&starturl.Module{URL: "http://www.example.com"},
		&proxy.Module{
			Proxies: []string{p.URL},
		},
	}

	scraper := flyscrape.NewScraper()
	scraper.Modules = mods
	scraper.Run()

	require.True(t, called)
}

func TestProxyMultiple(t *testing.T) {
	calls := []int{0, 0, 0}
	p0 := newProxy(func() { calls[0]++ })
	p1 := newProxy(func() { calls[1]++ })
	p2 := newProxy(func() { calls[2]++ })
	defer p0.Close()
	defer p1.Close()
	defer p2.Close()

	mod := &proxy.Module{Proxies: []string{p0.URL, p1.URL}, Proxy: p2.URL}
	mod.Provision(nil)
	trans := mod.AdaptTransport(nil)

	req := httptest.NewRequest("GET", "http://www.example.com/", nil)

	for i := 0; i < 50; i++ {
		resp, err := trans.RoundTrip(req)
		require.NoError(t, err)
		require.Equal(t, http.StatusOK, resp.StatusCode)
	}

	require.Greater(t, calls[0], 1)
	require.Greater(t, calls[1], 1)
	require.Greater(t, calls[2], 1)
	require.Equal(t, 50, calls[0]+calls[1]+calls[2])
}

func newProxy(f func()) *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		f()
		w.Write([]byte("response from proxy"))
	}))
}