Reference Script

This script serves as a reference that show all features of Flyscrape and how to use them. Feel free to copy and paste this as a starter script.
Reference
import { parse } from "flyscrape";
+import { download } from "flyscrape/http";
+import http from "flyscrape/http";
+
+export const config = {
+    // Specify the URL to start scraping from.
+    url: "https://example.com/",
+
+    // Specify the multiple URLs to start scraping from.   (default = [])
+    urls: [                          
+        "https://anothersite.com/",
+        "https://yetanother.com/",
+    ],
+
+    // Specify how deep links should be followed.          (default = 0, no follow)
+    depth: 5,                        
+
+    // Speficy the css selectors to follow.                (default = ["a[href]"])
+    follow: [".next > a", ".related a"],                      
+ 
+    // Specify the allowed domains. ['*'] for all.         (default = domain from url)
+    allowedDomains: ["example.com", "anothersite.com"],              
+ 
+    // Specify the blocked domains.                        (default = none)
+    blockedDomains: ["somesite.com"],              
+
+    // Specify the allowed URLs as regex.                  (default = all allowed)
+    allowedURLs: ["/posts", "/articles/\d+"],                 
+ 
+    // Specify the blocked URLs as regex.                  (default = none)
+    blockedURLs: ["/admin"],                 
+   
+    // Specify the rate in requests per minute.            (default = no rate limit)
+    rate: 60,                       
+
+    // Specify the number of concurrent requests.          (default = no limit)
+    concurrency: 1,                       
+
+    // Specify a single HTTP(S) proxy URL.                 (default = no proxy)
+    proxy: "http://someproxy.com:8043",
+
+    // Specify multiple HTTP(S) proxy URLs.                (default = no proxy)
+    proxies: [
+      "http://someproxy.com:8043",
+      "http://someotherproxy.com:8043",
+    ],                     
+
+    // Enable file-based request caching.                  (default = no cache)
+    cache: "file",                   
+
+    // Specify the HTTP request header.                    (default = none)
+    headers: {                       
+        "Authorization": "Bearer ...",
+        "User-Agent": "Mozilla ...",
+    },
+
+    // Specify the output options.
+    output: {
+        // Specify the output file.                        (default = stdout)
+        file: "results.json",
+        
+        // Specify the output format.                      (default = json)
+        // Options: "json" | "ndjson"
+        format: "json",
+    },
+};
+
+// Optional setup function, called before scraping starts.
+export function setup() {
+    // Fetch login form.
+    const { body } = http.get("http://example.com/login");
+
+    // Extract csrf token from form.
+    const csrf = parse(body).find("input[name=csrf]").attr("value");
+
+    // Submit login form.
+    http.postForm("http://example.com/login", {
+      "username": "jondoe",
+      "password": "supersecret",
+      "csrf": csrf,
+    });
+}
+
+export default function ({ doc, url, absoluteURL }) {
+  // doc              - Contains the parsed HTML document
+  // url              - Contains the scraped URL
+  // absoluteURL(...) - Transforms relative URLs into absolute URLs
+
+  // Find all users.
+  const userlist = doc.find(".user")
+
+  // Download the profile picture of each user.
+  userlist.each(user => {
+    const name = user.find(".name").text()
+    const pictureURL = absoluteURL(user.find("img").attr("src"));
+
+    download(pictureURL, `profile-pictures/${name}.jpg`)
+  })
+
+  // Return users name, address and age.
+  return {
+    users: userlist.map(user => {
+      const name = user.find(".name").text()
+      const address = user.find(".address").text()
+      const age = user.find(".age").text()
+
+      return { name, address, age };
+    })
+  };
+}

+    
+  

+
+ +
Getting Started API Reference
+ +