1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package flyscrape
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
gourl "net/url"
"strings"
)
func NewJSLibrary(client *http.Client) Imports {
return Imports{
"flyscrape": map[string]any{
"parse": jsParse(),
},
"flyscrape/http": map[string]any{
"get": jsHTTPGet(client),
"postForm": jsHTTPPostForm(client),
"postJSON": jsHTTPPostJSON(client),
},
}
}
func jsParse() func(html string) map[string]any {
return func(html string) map[string]any {
doc, err := DocumentFromString(html)
if err != nil {
return nil
}
return doc
}
}
func jsHTTPGet(client *http.Client) func(url string) map[string]any {
return func(url string) map[string]any {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return map[string]any{"error": err.Error()}
}
return jsFetch(client, req)
}
}
func jsHTTPPostForm(client *http.Client) func(url string, form map[string]any) map[string]any {
return func(url string, form map[string]any) map[string]any {
vals := gourl.Values{}
for k, v := range form {
switch v := v.(type) {
case []any:
for _, v := range v {
vals.Add(k, fmt.Sprintf("%v", v))
}
default:
vals.Add(k, fmt.Sprintf("%v", v))
}
}
req, err := http.NewRequest("POST", url, strings.NewReader(vals.Encode()))
if err != nil {
return map[string]any{"error": err.Error()}
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
return jsFetch(client, req)
}
}
func jsHTTPPostJSON(client *http.Client) func(url string, data any) map[string]any {
return func(url string, data any) map[string]any {
b, _ := json.Marshal(data)
req, err := http.NewRequest("POST", url, bytes.NewReader(b))
if err != nil {
return map[string]any{"error": err.Error()}
}
req.Header.Set("Content-Type", "application/json")
return jsFetch(client, req)
}
}
func jsFetch(client *http.Client, req *http.Request) (obj map[string]any) {
obj = map[string]any{
"body": "",
"status": 0,
"headers": map[string]any{},
"error": "",
}
resp, err := client.Do(req)
if err != nil {
obj["error"] = err.Error()
return
}
defer resp.Body.Close()
obj["status"] = resp.StatusCode
b, err := io.ReadAll(resp.Body)
if err != nil {
obj["error"] = err.Error()
return
}
obj["body"] = string(b)
headers := map[string]any{}
for name := range resp.Header {
headers[name] = resp.Header.Get(name)
}
obj["headers"] = headers
return
}
|