path: root/modules/proxy/proxy_test.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package proxy_test

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/philippta/flyscrape"
	"github.com/philippta/flyscrape/modules/proxy"
	"github.com/philippta/flyscrape/modules/starturl"
	"github.com/stretchr/testify/require"
)

func TestProxy(t *testing.T) {
	var called bool
	p := newProxy(func() { called = true })
	defer p.Close()

	scraper := flyscrape.NewScraper()
	scraper.LoadModule(&starturl.Module{URL: "http://www.example.com"})
	scraper.LoadModule(&proxy.Module{
		Proxies: []string{p.URL},
	})

	scraper.Run()
	require.True(t, called)
}

func TestProxyMultiple(t *testing.T) {
	calls := []int{0, 0}
	p0 := newProxy(func() { calls[0]++ })
	p1 := newProxy(func() { calls[1]++ })
	defer p0.Close()
	defer p1.Close()

	mod := &proxy.Module{Proxies: []string{p0.URL, p1.URL}}
	mod.Provision(nil)
	trans := mod.AdaptTransport(nil)

	req := httptest.NewRequest("GET", "http://www.example.com/", nil)

	for i := 0; i < 10; i++ {
		resp, err := trans.RoundTrip(req)
		require.NoError(t, err)
		require.Equal(t, http.StatusOK, resp.StatusCode)
	}

	require.Greater(t, calls[0], 1)
	require.Greater(t, calls[1], 1)
	require.Equal(t, 10, calls[0]+calls[1])
}

func newProxy(f func()) *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		f()
		w.Write([]byte("response from proxy"))
	}))
}