From 48ff300980369d6c06729a3e15575b77365ff59e Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Mon, 28 Aug 2023 18:21:30 +0200 Subject: add proxy to template --- js/template.js | 1 + js_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/js/template.js b/js/template.js index 75a60c7..2079c5f 100644 --- a/js/template.js +++ b/js/template.js @@ -7,6 +7,7 @@ export const options = { blockedDomains: [], // Specify the blocked domains. (default = none) allowedURLs: [], // Specify the allowed URLs as regex. (default = all allowed) blockedURLs: [], // Specify the blocked URLs as regex. (default = non blocked) + proxy: '', // Specify the HTTP(S) proxy to use. (default = no proxy) rate: 100, // Specify the rate in requests per second. (default = 100) } diff --git a/js_test.go b/js_test.go index d8ab305..3dd2873 100644 --- a/js_test.go +++ b/js_test.go @@ -75,3 +75,31 @@ func TestJSCompileError(t *testing.T) { Text: `Expected "from" but found ";"`, }) } + +func TestJSOptions(t *testing.T) { + js := ` + export const options = { + url: 'http://localhost/', + depth: 5, + allowedDomains: ['example.com'], + blockedDomains: ['google.com'], + allowedURLs: ['/foo'], + blockedURLs: ['/bar'], + proxy: 'http://proxy/', + rate: 1, + } + export default function() {} + ` + opts, _, err := flyscrape.Compile(js) + require.NoError(t, err) + require.Equal(t, flyscrape.ScrapeOptions{ + URL: "http://localhost/", + Depth: 5, + AllowedDomains: []string{"example.com"}, + BlockedDomains: []string{"google.com"}, + AllowedURLs: []string{"/foo"}, + BlockedURLs: []string{"/bar"}, + Proxy: "http://proxy/", + Rate: 1, + }, opts) +} -- cgit v1.2.3