From b1e2c8fd5cb5dfa46bc440a12eafaf56cd844b1c Mon Sep 17 00:00:00 2001 From: Philipp Tanlak Date: Mon, 24 Nov 2025 20:54:57 +0100 Subject: Docs --- public/docs/configuration/browser-mode/index.html | 612 ++++++++++++++++++ public/docs/configuration/caching/index.html | 604 ++++++++++++++++++ public/docs/configuration/concurrency/index.html | 560 +++++++++++++++++ public/docs/configuration/cookies/index.html | 595 ++++++++++++++++++ public/docs/configuration/depth/index.html | 577 +++++++++++++++++ public/docs/configuration/domain-filter/index.html | 616 ++++++++++++++++++ public/docs/configuration/headers/index.html | 557 +++++++++++++++++ public/docs/configuration/index.html | 561 +++++++++++++++++ public/docs/configuration/index.xml | 686 +++++++++++++++++++++ .../docs/configuration/link-following/index.html | 593 ++++++++++++++++++ public/docs/configuration/output/index.html | 615 ++++++++++++++++++ public/docs/configuration/proxies/index.html | 591 ++++++++++++++++++ public/docs/configuration/rate-limiting/index.html | 559 +++++++++++++++++ public/docs/configuration/retry/index.html | 576 +++++++++++++++++ public/docs/configuration/starting-url/index.html | 589 ++++++++++++++++++ public/docs/configuration/url-filter/index.html | 610 ++++++++++++++++++ 16 files changed, 9501 insertions(+) create mode 100644 public/docs/configuration/browser-mode/index.html create mode 100644 public/docs/configuration/caching/index.html create mode 100644 public/docs/configuration/concurrency/index.html create mode 100644 public/docs/configuration/cookies/index.html create mode 100644 public/docs/configuration/depth/index.html create mode 100644 public/docs/configuration/domain-filter/index.html create mode 100644 public/docs/configuration/headers/index.html create mode 100644 public/docs/configuration/index.html create mode 100644 public/docs/configuration/index.xml create mode 100644 public/docs/configuration/link-following/index.html create mode 100644 public/docs/configuration/output/index.html create mode 100644 public/docs/configuration/proxies/index.html create mode 100644 public/docs/configuration/rate-limiting/index.html create mode 100644 public/docs/configuration/retry/index.html create mode 100644 public/docs/configuration/starting-url/index.html create mode 100644 public/docs/configuration/url-filter/index.html (limited to 'public/docs/configuration') diff --git a/public/docs/configuration/browser-mode/index.html b/public/docs/configuration/browser-mode/index.html new file mode 100644 index 0000000..123458e --- /dev/null +++ b/public/docs/configuration/browser-mode/index.html @@ -0,0 +1,612 @@ + + + + + + + + + + + + + + + Browser Mode – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Browser Mode
+
+ +
+

Browser Mode

+

The Browser Mode controls the interaction with a headless Chromium browser. Enabling the browser mode allows flyscrape to download a Chromium browser once and use it to render JavaScript-heavy pages.

+

Browser Mode +

To enable Browser Mode, set the browser option to true in your configuration. This allows flyscrape to use a headless Chromium browser for rendering JavaScript during the scraping process.

+
Configuration
export const config = {
+    browser: true,
+};
+ +
+
+

In the above example, Browser Mode is enabled, allowing flyscrape to render pages that rely on JavaScript execution.

+

Headless Option +

The headless option, when combined with Browser Mode, controls whether the Chromium browser should run in headless mode or not. Headless mode means the browser operates without a graphical user interface, which can be useful for background processes.

+
Configuration
export const config = {
+    browser: true,
+    headless: false,
+};
+ +
+
+

In this example, the Chromium browser will run in non-headless mode. If you set headless to true, the browser will run without a visible GUI.

+
Configuration
export const config = {
+    browser: true,
+    headless: true,
+};
+ +
+
+

In this example, the Chromium browser will run in headless mode, suitable for scenarios where graphical rendering is unnecessary.

+ +
+
+
HeadersOutput File and Format
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/caching/index.html b/public/docs/configuration/caching/index.html new file mode 100644 index 0000000..127eb18 --- /dev/null +++ b/public/docs/configuration/caching/index.html @@ -0,0 +1,604 @@ + + + + + + + + + + + + + + + Caching – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Caching
+
+ +
+

Caching

+

The cache config option allows you to enable file-based request caching. When enabled every request cached with its raw response. When the cache is populated and you re-run the scraper, requests will be served directly from cache.

+

This also allows you to modify your scraping script afterwards and collect new results immediately.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  cache: "file",
+  // ...
+};
+ +
+
+

Cache File +

When caching is enabled using the cache: "file" option, a .cache file will be created with the name of your scraping script.

+
Terminal
$ flyscrape run hackernews.js # Will populate: hackernews.cache
+ +
+
+

Shared cache +

In case you want to share a cache between different scraping scripts, you can specify where to store the cache file.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  cache: "file:/some/path/shared.cache",
+  // ...
+};
+ +
+
+ +
+
+
RetryProxies
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/concurrency/index.html b/public/docs/configuration/concurrency/index.html new file mode 100644 index 0000000..0047445 --- /dev/null +++ b/public/docs/configuration/concurrency/index.html @@ -0,0 +1,560 @@ + + + + + + + + + + + + + + + Concurrency – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Concurrency
+
+ +
+

Concurrency

+

The concurrency setting controls the number of simultaneous requests that the scraper can make. This is specified in the configuration object of your scraping script.

+
export const config = {
+    // Specify the number of concurrent requests.
+    concurrency: 5,
+};
+ +
+
+

In the above example, the scraper will make up to 5 requests at the same time.

+

If the concurrency setting is not specified, there is no limit to the number of concurrent requests.

+ +
+
+
Link FollowingRate Limiting
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/cookies/index.html b/public/docs/configuration/cookies/index.html new file mode 100644 index 0000000..d514e9b --- /dev/null +++ b/public/docs/configuration/cookies/index.html @@ -0,0 +1,595 @@ + + + + + + + + + + + + + + + Cookies – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Cookies
+
+ +
+

Cookies

+

The Cookies configuration in the flyscrape script’s configuration object allows you to specify the behavior of the cookie store during the scraping process. Cookies are often used for authentication and session management on websites.

+

Cookies Configuration +

To configure the cookie store behavior, set the cookies field in your configuration. The cookies option supports three values: "chrome", "edge", and "firefox". Each value corresponds to using the cookie store of the respective local browser.

+

When the cookies option is set to "chrome", "edge", or "firefox", flyscrape utilizes the cookie store of the user’s installed browser.

+
Configuration
export const config = {
+    cookies: "chrome",
+};
+ +
+
+

In the above example, the cookies option is set to "chrome", indicating that flyscrape should use the cookie store of the local Chrome browser.

+
Configuration
export const config = {
+    cookies: "firefox",
+};
+ +
+
+

In this example, the cookies option is set to "firefox", instructing flyscrape to use the cookie store of the local Firefox browser.

+
Configuration
export const config = {
+    cookies: "edge",
+};
+ +
+
+

In this example, the cookies option is set to "edge", indicating that flyscrape should use the cookie store of the local Edge browser.

+ +
+
+
ProxiesHeaders
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/depth/index.html b/public/docs/configuration/depth/index.html new file mode 100644 index 0000000..ac94970 --- /dev/null +++ b/public/docs/configuration/depth/index.html @@ -0,0 +1,577 @@ + + + + + + + + + + + + + + + Depth – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Depth
+
+ +
+

Depth

+

The depth config option allows you to specify how deep the scraping process should follow links from the initial URL.

+

When no value is provided or depth is set to 0 link following is disabled and it will only scrape the initial URL.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  depth: 2,
+  // ...
+};
+ +
+
+

With the config provided in the example the scraper would follow links like this:

+
http://example.com/                    (depth = 0, initial URL)
+↳ http://example.com/deeply            (depth = 1)
+  ↳ http://example.com/deeply/nested   (depth = 2)
+ +
+
+ +
+
+
Starting URLDomain Filter
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/domain-filter/index.html b/public/docs/configuration/domain-filter/index.html new file mode 100644 index 0000000..482211f --- /dev/null +++ b/public/docs/configuration/domain-filter/index.html @@ -0,0 +1,616 @@ + + + + + + + + + + + + + + + Domain Filter – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Domain Filter
+
+ +
+

Domain Filter

+

The allowedDomains and blockedDomains config options allow you to specify a list of domains which are accessible or blocked during scraping.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  allowedDomains: ["subdomain.example.com"],
+  // ...
+};
+ +
+
+

Allowed Domains +

This config option controls which additional domains are allowed to be visted during scraping. The domain of the initial URL is always allowed.

+

You can also allow all domains to be accessible by setting allowedDomains to ["*"]. To then further restrict access, you can specify blockedDomains.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  allowedDomains: ["*"],
+  // ...
+};
+ +
+
+

Blocked Domains +

This config option controls which additional domains are blocked from being accessed. By default all domains other than the domain of the initial URL or those specified in allowedDomains are blocked.

+

You can best use blockedDomains in conjunction with allowedDomains: ["*"], allowing the scraping process to access all domains except what’s specified in blockedDomains.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  allowedDomains: ["*"],
+  blockedDomains: ["google.com", "bing.com"],
+  // ...
+};
+ +
+
+ +
+
+
DepthURL Filter
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/headers/index.html b/public/docs/configuration/headers/index.html new file mode 100644 index 0000000..b4e034f --- /dev/null +++ b/public/docs/configuration/headers/index.html @@ -0,0 +1,557 @@ + + + + + + + + + + + + + + + Headers – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Headers
+
+ +
+

Headers

+

The headers config option allows you to specify the custom HTTP headers sent with each request.

+
Configuration
export const config = {
+  headers: {
+    "Authorization": "Bearer ey....",
+    "User-Agent": "Mozilla/5.0 (Macintosh ...",
+  },
+  // ...
+};
+ +
+
+ +
+
+
CookiesBrowser Mode
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/index.html b/public/docs/configuration/index.html new file mode 100644 index 0000000..75a7649 --- /dev/null +++ b/public/docs/configuration/index.html @@ -0,0 +1,561 @@ + + + + + + + + + + + + + + + Configuration – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + + +
+ + + + + + + + + + diff --git a/public/docs/configuration/index.xml b/public/docs/configuration/index.xml new file mode 100644 index 0000000..85c2f0d --- /dev/null +++ b/public/docs/configuration/index.xml @@ -0,0 +1,686 @@ + + + Flyscrape – Configuration + https://flyscrape.com/docs/configuration/ + Recent content in Configuration on Flyscrape + Hugo -- gohugo.io + en-us + + + + + + + + + + + Starting URL + https://flyscrape.com/docs/configuration/starting-url/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/starting-url/ + + + + <p>The <code>url</code> config option allows you to specify the initial URL at which the scraper should start its scraping process.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Multiple starting URLs<span class="absolute -mt-20" id="multiple-starting-urls"></span> + <a href="#multiple-starting-urls" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>In case you have more than one URL you want to scrape (or to start from) you can specify them with the <code>urls</code> config option.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">urls</span><span class="o">:</span> <span class="p">[</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;http://anothersite.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;http://yetanothersite.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Depth + https://flyscrape.com/docs/configuration/depth/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/depth/ + + + + <p>The <code>depth</code> config option allows you to specify how deep the scraping process should follow links from the initial URL.</p> +<p>When no value is provided or <code>depth</code> is set to <code>0</code> link following is disabled and it will only scrape the initial URL.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">depth</span><span class="o">:</span> <span class="mi">2</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>With the config provided in the example the scraper would follow links like this:</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><pre><code>http://example.com/ (depth = 0, initial URL) +↳ http://example.com/deeply (depth = 1) + ↳ http://example.com/deeply/nested (depth = 2)</code></pre><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-0"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Domain Filter + https://flyscrape.com/docs/configuration/domain-filter/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/domain-filter/ + + + + <p>The <code>allowedDomains</code> and <code>blockedDomains</code> config options allow you to specify a list of domains which are accessible or blocked during scraping.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">allowedDomains</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;subdomain.example.com&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Allowed Domains<span class="absolute -mt-20" id="allowed-domains"></span> + <a href="#allowed-domains" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>This config option controls which additional domains are allowed to be visted during scraping. The domain of the initial URL is always allowed.</p> +<p>You can also allow all domains to be accessible by setting <code>allowedDomains</code> to <code>[&quot;*&quot;]</code>. To then further restrict access, you can specify <code>blockedDomains</code>.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">allowedDomains</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;*&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Blocked Domains<span class="absolute -mt-20" id="blocked-domains"></span> + <a href="#blocked-domains" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>This config option controls which additional domains are blocked from being accessed. By default all domains other than the domain of the initial URL or those specified in <code>allowedDomains</code> are blocked.</p> +<p>You can best use <code>blockedDomains</code> in conjunction with <code>allowedDomains: [&quot;*&quot;]</code>, allowing the scraping process to access all domains except what&rsquo;s specified in <code>blockedDomains</code>.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">allowedDomains</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;*&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="nx">blockedDomains</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;google.com&#34;</span><span class="p">,</span> <span class="s2">&#34;bing.com&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + URL Filter + https://flyscrape.com/docs/configuration/url-filter/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/url-filter/ + + + + <p>The <code>allowedURLs</code> and <code>blockedURLs</code> config options allow you to specify a list of URL patterns (in form of regular expressions) which are accessible or blocked during scraping.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">allowedURLs</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;/articles/.*&#34;</span><span class="p">,</span> <span class="s2">&#34;/authors/.*&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="nx">blockedURLs</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;/authors/admin&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Allowed URLs<span class="absolute -mt-20" id="allowed-urls"></span> + <a href="#allowed-urls" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>This config option controls which URLs are allowed to be visted during scraping. When no value is provided all URLs are allowed to be visited if not otherwise blocked.</p> +<p>When a list of URL patterns is provided, only URLs matching one or more of these patterns are allowed to be visted.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">allowedURLs</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;/products/&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Blocked URLs<span class="absolute -mt-20" id="blocked-urls"></span> + <a href="#blocked-urls" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>This config option controls which URLs are blocked from being visted during scraping.</p> +<p>When a list of URL patterns is provided, URLs matching one or more of these patterns are blocked from to be visted.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">blockedURLs</span><span class="o">:</span> <span class="p">[</span><span class="s2">&#34;/restricted&#34;</span><span class="p">],</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Link Following + https://flyscrape.com/docs/configuration/link-following/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/link-following/ + + + + <p>The <code>follow</code> config option allows you to specify a list of CSS selectors that determine which links the scraper should follow.</p> +<p>When no value is provided the scraper will follow all links found with the <code>a[href]</code> selector.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">follow</span><span class="o">:</span> <span class="p">[</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;.pagination &gt; a[href]&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;.nav a[href]&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h2>Following non <code>href</code> attributes<span class="absolute -mt-20" id="following-non-href-attributes"></span> + <a href="#following-non-href-attributes" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>For special cases where the link is not to be found in the <code>href</code>, you specify a selector with a different ending attribute.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">follow</span><span class="o">:</span> <span class="p">[</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;.articles &gt; div[data-url]&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">],</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Concurrency + https://flyscrape.com/docs/configuration/concurrency/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/concurrency/ + + + + <p>The concurrency setting controls the number of simultaneous requests that the scraper can make. This is specified in the configuration object of your scraping script.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify the number of concurrent requests. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">concurrency</span><span class="o">:</span> <span class="mi">5</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-0"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, the scraper will make up to 5 requests at the same time.</p> +<p>If the concurrency setting is not specified, there is no limit to the number of concurrent requests.</p> + + + + + + Rate Limiting + https://flyscrape.com/docs/configuration/rate-limiting/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/rate-limiting/ + + + + <p>The <code>rate</code> config option allows you to specify at which rate the scraper should send out requests. The rate is measured in <em>Requests per Minute</em> (RPM).</p> +<p>When no <code>rate</code> is specified, rate limiting is disabled and the scraper will send out requests as fast as it can.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">options</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">rate</span><span class="o">:</span> <span class="mi">100</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Retry + https://flyscrape.com/docs/configuration/retry/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/retry/ + + + + <p>The retry feature allows the scraper to automatically retry failed requests. This is particularly useful when dealing with unstable networks or servers that occasionally return error status codes.</p> +<p>The retry feature is automatically enabled and will retry requests that return the following HTTP status codes:</p> +<ul> +<li>403 Forbidden</li> +<li>408 Request Timeout</li> +<li>425 Too Early</li> +<li>429 Too Many Requests</li> +<li>500 Internal Server Error</li> +<li>502 Bad Gateway</li> +<li>503 Service Unavailable</li> +<li>504 Gateway Timeout</li> +</ul> +<h3>Retry Delays<span class="absolute -mt-20" id="retry-delays"></span> + <a href="#retry-delays" class="subheading-anchor" aria-label="Permalink for this section"></a></h3><p>After a failed request, the scraper will wait for a certain amount of time before retrying the request. The delay increases with each consecutive failed attempt, according to the following schedule:</p> +<ul> +<li>1st retry: 1 second delay</li> +<li>2nd retry: 2 seconds delay</li> +<li>3rd retry: 5 seconds delay</li> +<li>4th retry: 10 seconds delay</li> +</ul> + + + + + + Caching + https://flyscrape.com/docs/configuration/caching/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/caching/ + + + + <p>The <code>cache</code> config option allows you to enable file-based request caching. When enabled every request cached with its raw response. When the cache is populated and you re-run the scraper, requests will be served directly from cache.</p> +<p>This also allows you to modify your scraping script afterwards and collect new results immediately.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">cache</span><span class="o">:</span> <span class="s2">&#34;file&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h3>Cache File<span class="absolute -mt-20" id="cache-file"></span> + <a href="#cache-file" class="subheading-anchor" aria-label="Permalink for this section"></a></h3><p>When caching is enabled using the <code>cache: &quot;file&quot;</code> option, a <code>.cache</code> file will be created with the name of your scraping script.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Terminal</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"><span class="line"><span class="cl">$ flyscrape run hackernews.js <span class="c1"># Will populate: hackernews.cache</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<h3>Shared cache<span class="absolute -mt-20" id="shared-cache"></span> + <a href="#shared-cache" class="subheading-anchor" aria-label="Permalink for this section"></a></h3><p>In case you want to share a cache between different scraping scripts, you can specify where to store the cache file.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://example.com/&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">cache</span><span class="o">:</span> <span class="s2">&#34;file:/some/path/shared.cache&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Proxies + https://flyscrape.com/docs/configuration/proxies/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/proxies/ + + + + <p>The proxy feature allows you to route your scraping requests through a specified HTTP(S) proxy. This can be useful for bypassing IP-based rate limits or accessing region-restricted content.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify a single HTTP(S) proxy URL. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">proxy</span><span class="o">:</span> <span class="s2">&#34;http://someproxy.com:8043&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-0"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, all scraping requests will be routed through the proxy at <code>http://someproxy.com:8043</code>.</p> +<h2>Multiple Proxies<span class="absolute -mt-20" id="multiple-proxies"></span> + <a href="#multiple-proxies" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>You can also specify multiple proxy URLs. The scraper will rotate between these proxies for each request.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify multiple HTTP(S) proxy URLs. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">proxies</span><span class="o">:</span> <span class="p">[</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;http://someproxy.com:8043&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;http://someotherproxy.com:8043&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">],</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-0"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the scraper will randomly pick between the proxies at <code>http://someproxy.com:8043</code> and <code>http://someotherproxy.com:8043</code>.</p> +<p>Note: If both <code>proxy</code> and <code>proxies</code> are specified, all proxies will be respected.</p> + + + + + + Cookies + https://flyscrape.com/docs/configuration/cookies/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/cookies/ + + + + <p>The Cookies configuration in the <code>flyscrape</code> script&rsquo;s configuration object allows you to specify the behavior of the cookie store during the scraping process. Cookies are often used for authentication and session management on websites.</p> +<h2>Cookies Configuration<span class="absolute -mt-20" id="cookies-configuration"></span> + <a href="#cookies-configuration" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>To configure the cookie store behavior, set the <code>cookies</code> field in your configuration. The <code>cookies</code> option supports three values: <code>&quot;chrome&quot;</code>, <code>&quot;edge&quot;</code>, and <code>&quot;firefox&quot;</code>. Each value corresponds to using the cookie store of the respective local browser.</p> +<p>When the <code>cookies</code> option is set to <code>&quot;chrome&quot;</code>, <code>&quot;edge&quot;</code>, or <code>&quot;firefox&quot;</code>, <code>flyscrape</code> utilizes the cookie store of the user&rsquo;s installed browser.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">cookies</span><span class="o">:</span> <span class="s2">&#34;chrome&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, the <code>cookies</code> option is set to <code>&quot;chrome&quot;</code>, indicating that <code>flyscrape</code> should use the cookie store of the local Chrome browser.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">cookies</span><span class="o">:</span> <span class="s2">&#34;firefox&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the <code>cookies</code> option is set to <code>&quot;firefox&quot;</code>, instructing <code>flyscrape</code> to use the cookie store of the local Firefox browser.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">cookies</span><span class="o">:</span> <span class="s2">&#34;edge&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the <code>cookies</code> option is set to <code>&quot;edge&quot;</code>, indicating that <code>flyscrape</code> should use the cookie store of the local Edge browser.</p> + + + + + + Headers + https://flyscrape.com/docs/configuration/headers/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/headers/ + + + + <p>The <code>headers</code> config option allows you to specify the custom HTTP headers sent with each request.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">headers</span><span class="o">:</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;Authorization&#34;</span><span class="o">:</span> <span class="s2">&#34;Bearer ey....&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="s2">&#34;User-Agent&#34;</span><span class="o">:</span> <span class="s2">&#34;Mozilla/5.0 (Macintosh ...&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">},</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// ... +</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> + + + + + + Browser Mode + https://flyscrape.com/docs/configuration/browser-mode/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/browser-mode/ + + + + <p>The Browser Mode controls the interaction with a headless Chromium browser. Enabling the browser mode allows <code>flyscrape</code> to download a Chromium browser once and use it to render JavaScript-heavy pages.</p> +<h2>Browser Mode<span class="absolute -mt-20" id="browser-mode"></span> + <a href="#browser-mode" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>To enable Browser Mode, set the <code>browser</code> option to <code>true</code> in your configuration. This allows <code>flyscrape</code> to use a headless Chromium browser for rendering JavaScript during the scraping process.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">browser</span><span class="o">:</span> <span class="kc">true</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, Browser Mode is enabled, allowing <code>flyscrape</code> to render pages that rely on JavaScript execution.</p> +<h2>Headless Option<span class="absolute -mt-20" id="headless-option"></span> + <a href="#headless-option" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>The <code>headless</code> option, when combined with Browser Mode, controls whether the Chromium browser should run in headless mode or not. Headless mode means the browser operates without a graphical user interface, which can be useful for background processes.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">browser</span><span class="o">:</span> <span class="kc">true</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">headless</span><span class="o">:</span> <span class="kc">false</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the Chromium browser will run in non-headless mode. If you set <code>headless</code> to <code>true</code>, the browser will run without a visible GUI.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">browser</span><span class="o">:</span> <span class="kc">true</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="nx">headless</span><span class="o">:</span> <span class="kc">true</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the Chromium browser will run in headless mode, suitable for scenarios where graphical rendering is unnecessary.</p> + + + + + + Output File and Format + https://flyscrape.com/docs/configuration/output/ + Mon, 01 Jan 0001 00:00:00 +0000 + + https://flyscrape.com/docs/configuration/output/ + + + + <p>The output file and format are specified in the configuration object of your scraping script. They determine where the scraped data will be saved and in what format.</p> +<h2>Output File<span class="absolute -mt-20" id="output-file"></span> + <a href="#output-file" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>The output file is the file where the scraped data will be saved. If not specified, the data will be printed to the standard output (stdout).</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">output</span><span class="o">:</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify the output file. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">file</span><span class="o">:</span> <span class="s2">&#34;results.json&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">},</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, the scraped data will be saved in a file named <code>results.json</code>.</p> +<h2>Output Format<span class="absolute -mt-20" id="output-format"></span> + <a href="#output-format" class="subheading-anchor" aria-label="Permalink for this section"></a></h2><p>The output format is the format in which the scraped data will be saved. The options are <code>json</code> and <code>ndjson</code>.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">output</span><span class="o">:</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify the output format. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">format</span><span class="o">:</span> <span class="s2">&#34;json&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">},</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In the above example, the scraped data will be saved in JSON format.</p> +<div class="code-block relative mt-6 first:mt-0 group/code"><div class="filename">Configuration</div><div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-javascript" data-lang="javascript"><span class="line"><span class="cl"><span class="kr">export</span> <span class="kr">const</span> <span class="nx">config</span> <span class="o">=</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="nx">output</span><span class="o">:</span> <span class="p">{</span> +</span></span><span class="line"><span class="cl"> <span class="c1">// Specify the output format. +</span></span></span><span class="line"><span class="cl"><span class="c1"></span> <span class="nx">format</span><span class="o">:</span> <span class="s2">&#34;ndjson&#34;</span><span class="p">,</span> +</span></span><span class="line"><span class="cl"> <span class="p">},</span> +</span></span><span class="line"><span class="cl"><span class="p">};</span></span></span></code></pre></div></div><div class="opacity-0 transition group-hover/code:opacity-100 flex gap-1 absolute m-[11px] right-0 top-8"> + <button + class="code-copy-btn group/copybtn transition-all active:opacity-50 bg-primary-700/5 border border-black/5 text-gray-600 hover:text-gray-900 rounded-md p-1.5 dark:bg-primary-300/10 dark:border-white/10 dark:text-gray-400 dark:hover:text-gray-50" + title="Copy code" + > + <div class="group-[.copied]/copybtn:hidden copy-icon pointer-events-none h-4 w-4"></div> + <div class="hidden group-[.copied]/copybtn:block success-icon pointer-events-none h-4 w-4"></div> + </button> + </div> +</div> +<p>In this example, the scraped data will be saved in newline-delimited JSON (NDJSON) format. Each line in the output file will be a separate JSON object.</p> + + + + + + diff --git a/public/docs/configuration/link-following/index.html b/public/docs/configuration/link-following/index.html new file mode 100644 index 0000000..46434c3 --- /dev/null +++ b/public/docs/configuration/link-following/index.html @@ -0,0 +1,593 @@ + + + + + + + + + + + + + + + Link Following – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Link Following
+
+ +
+

Link Following

+

The follow config option allows you to specify a list of CSS selectors that determine which links the scraper should follow.

+

When no value is provided the scraper will follow all links found with the a[href] selector.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  follow: [
+    ".pagination > a[href]",
+    ".nav a[href]",
+  ],
+  // ...
+};
+ +
+
+

Following non href attributes +

For special cases where the link is not to be found in the href, you specify a selector with a different ending attribute.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  follow: [
+    ".articles > div[data-url]",
+  ],
+  // ...
+};
+ +
+
+ +
+
+
URL FilterConcurrency
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/output/index.html b/public/docs/configuration/output/index.html new file mode 100644 index 0000000..4d90b23 --- /dev/null +++ b/public/docs/configuration/output/index.html @@ -0,0 +1,615 @@ + + + + + + + + + + + + + + + Output File and Format – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Output File and Format
+
+ +
+

Output File and Format

+

The output file and format are specified in the configuration object of your scraping script. They determine where the scraped data will be saved and in what format.

+

Output File +

The output file is the file where the scraped data will be saved. If not specified, the data will be printed to the standard output (stdout).

+
Configuration
export const config = {
+    output: {
+        // Specify the output file.
+        file: "results.json",
+    },
+};
+ +
+
+

In the above example, the scraped data will be saved in a file named results.json.

+

Output Format +

The output format is the format in which the scraped data will be saved. The options are json and ndjson.

+
Configuration
export const config = {
+    output: {
+        // Specify the output format.
+        format: "json",
+    },
+};
+ +
+
+

In the above example, the scraped data will be saved in JSON format.

+
Configuration
export const config = {
+    output: {
+        // Specify the output format.
+        format: "ndjson",
+    },
+};
+ +
+
+

In this example, the scraped data will be saved in newline-delimited JSON (NDJSON) format. Each line in the output file will be a separate JSON object.

+ +
+
+
Browser Mode
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/proxies/index.html b/public/docs/configuration/proxies/index.html new file mode 100644 index 0000000..3a43ad5 --- /dev/null +++ b/public/docs/configuration/proxies/index.html @@ -0,0 +1,591 @@ + + + + + + + + + + + + + + + Proxies – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Proxies
+
+ +
+

Proxies

+

The proxy feature allows you to route your scraping requests through a specified HTTP(S) proxy. This can be useful for bypassing IP-based rate limits or accessing region-restricted content.

+
export const config = {
+    // Specify a single HTTP(S) proxy URL.
+    proxy: "http://someproxy.com:8043",
+};
+ +
+
+

In the above example, all scraping requests will be routed through the proxy at http://someproxy.com:8043.

+

Multiple Proxies +

You can also specify multiple proxy URLs. The scraper will rotate between these proxies for each request.

+
export const config = {
+    // Specify multiple HTTP(S) proxy URLs.
+    proxies: [
+      "http://someproxy.com:8043",
+      "http://someotherproxy.com:8043",
+    ],                     
+};
+ +
+
+

In this example, the scraper will randomly pick between the proxies at http://someproxy.com:8043 and http://someotherproxy.com:8043.

+

Note: If both proxy and proxies are specified, all proxies will be respected.

+ +
+
+
CachingCookies
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/rate-limiting/index.html b/public/docs/configuration/rate-limiting/index.html new file mode 100644 index 0000000..76d02b8 --- /dev/null +++ b/public/docs/configuration/rate-limiting/index.html @@ -0,0 +1,559 @@ + + + + + + + + + + + + + + + Rate Limiting – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Rate Limiting
+
+ +
+

Rate Limiting

+

The rate config option allows you to specify at which rate the scraper should send out requests. The rate is measured in Requests per Minute (RPM).

+

When no rate is specified, rate limiting is disabled and the scraper will send out requests as fast as it can.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  rate: 100,
+};
+ +
+
+ +
+
+
ConcurrencyRetry
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/retry/index.html b/public/docs/configuration/retry/index.html new file mode 100644 index 0000000..3da9826 --- /dev/null +++ b/public/docs/configuration/retry/index.html @@ -0,0 +1,576 @@ + + + + + + + + + + + + + + + Retry – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Retry
+
+ +
+

Retry

+

The retry feature allows the scraper to automatically retry failed requests. This is particularly useful when dealing with unstable networks or servers that occasionally return error status codes.

+

The retry feature is automatically enabled and will retry requests that return the following HTTP status codes:

+
    +
  • 403 Forbidden
    • +
  • 408 Request Timeout
    • +
  • 425 Too Early
    • +
  • 429 Too Many Requests
    • +
  • 500 Internal Server Error
    • +
  • 502 Bad Gateway
    • +
  • 503 Service Unavailable
    • +
  • 504 Gateway Timeout
    • +
+

Retry Delays +

After a failed request, the scraper will wait for a certain amount of time before retrying the request. The delay increases with each consecutive failed attempt, according to the following schedule:

+
    +
  • 1st retry: 1 second delay
    • +
  • 2nd retry: 2 seconds delay
    • +
  • 3rd retry: 5 seconds delay
    • +
  • 4th retry: 10 seconds delay
    • +
+ +
+
+
Rate LimitingCaching
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/starting-url/index.html b/public/docs/configuration/starting-url/index.html new file mode 100644 index 0000000..77130fb --- /dev/null +++ b/public/docs/configuration/starting-url/index.html @@ -0,0 +1,589 @@ + + + + + + + + + + + + + + + Starting URL – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
Starting URL
+
+ +
+

Starting URL

+

The url config option allows you to specify the initial URL at which the scraper should start its scraping process.

+
Configuration
export const config = {
+  url: "http://example.com/",
+  // ...
+};
+ +
+
+

Multiple starting URLs +

In case you have more than one URL you want to scrape (or to start from) you can specify them with the urls config option.

+
Configuration
export const config = {
+  urls: [
+    "http://example.com/",
+    "http://anothersite.com/",
+    "http://yetanothersite.com/",
+  ],
+  // ...
+};
+ +
+
+ +
+
+
ConfigurationDepth
+ +
+
+
+ + + + + + + + + + diff --git a/public/docs/configuration/url-filter/index.html b/public/docs/configuration/url-filter/index.html new file mode 100644 index 0000000..264627f --- /dev/null +++ b/public/docs/configuration/url-filter/index.html @@ -0,0 +1,610 @@ + + + + + + + + + + + + + + + URL Filter – Flyscrape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + + + + +
+
+
+
+ Documentation +
+
+ Configuration +
URL Filter
+
+ +
+

URL Filter

+

The allowedURLs and blockedURLs config options allow you to specify a list of URL patterns (in form of regular expressions) which are accessible or blocked during scraping.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  allowedURLs: ["/articles/.*", "/authors/.*"],
+  blockedURLs: ["/authors/admin"],
+  // ...
+};
+ +
+
+

Allowed URLs +

This config option controls which URLs are allowed to be visted during scraping. When no value is provided all URLs are allowed to be visited if not otherwise blocked.

+

When a list of URL patterns is provided, only URLs matching one or more of these patterns are allowed to be visted.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  allowedURLs: ["/products/"],
+};
+ +
+
+

Blocked URLs +

This config option controls which URLs are blocked from being visted during scraping.

+

When a list of URL patterns is provided, URLs matching one or more of these patterns are blocked from to be visted.

+
Configuration
export const options = {
+  url: "http://example.com/",
+  blockedURLs: ["/restricted"],
+};
+ +
+
+ +
+
+
Getting StartedLink Following
+ +
+
+
+ + + + + + + + + + -- cgit v1.2.3