From 190056ee8d6a4eca61d92a79cc25aad645e69d4a Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Mon, 13 Nov 2023 22:36:15 +0100 Subject: Move docs to flyscrape.com (#11) --- docs/configuration/url-filter.md | 42 ---------------------------------------- 1 file changed, 42 deletions(-) delete mode 100644 docs/configuration/url-filter.md (limited to 'docs/configuration/url-filter.md') diff --git a/docs/configuration/url-filter.md b/docs/configuration/url-filter.md deleted file mode 100644 index e2feda8..0000000 --- a/docs/configuration/url-filter.md +++ /dev/null @@ -1,42 +0,0 @@ -# URL Filter - -The `allowedURLs` and `blockedURLs` config options allow you to specify a list of URL patterns (in form of regular expressions) which are accessible or blocked during scraping. - -```javascript -export const options = { - url: "http://example.com/", - allowedURLs: ["/articles/.*", "/authors/.*"], - blockedURLs: ["/authors/admin"], - // ... -}; -``` - -### `allowedURLs` - -This config option controls which URLs are allowed to be visted during scraping. When no value is provided all URLs are allowed to be visited if not otherwise blocked. - -When a list of URL patterns is provided, only URLs matching one or more of these patterns are allowed to be visted. - -Example: - -```javascript -export const options = { - url: "http://example.com/", - allowedURLs: ["/products/"], -}; -``` - -### `blockedURLs` - -This config option controls which URLs are blocked from being visted during scraping. - -When a list of URL patterns is provided, URLs matching one or more of these patterns are blocked from to be visted. - -Example: - -```javascript -export const options = { - url: "http://example.com/", - blockedURLs: ["/restricted"], -}; -``` -- cgit v1.2.3