Generate random user agent when non is provided (#76)

10 files changed, 1650 insertions, 71 deletions

diff --git a/modules/headers/headers.go b/modules/headers/headers.go
index 877b370..5ed58f9 100644
--- a/modules/headers/headers.go
+++ b/modules/headers/headers.go
@@ -26,15 +26,15 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo {
 }
 
 func (m Module) AdaptTransport(t http.RoundTripper) http.RoundTripper {
-	if len(m.Headers) == 0 {
-		return t
-	}
-
 	return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
 		for k, v := range m.Headers {
 			r.Header.Set(k, v)
 		}
 
+		if r.Header.Get("User-Agent") == "" {
+			r.Header.Set("User-Agent", randomUserAgent())
+		}
+
 		return t.RoundTrip(r)
 	})
 }
diff --git a/modules/headers/headers_test.go b/modules/headers/headers_test.go
index 72b9001..afd8a72 100644
--- a/modules/headers/headers_test.go
+++ b/modules/headers/headers_test.go
@@ -5,9 +5,8 @@
 package headers_test
 
 import (
-	"fmt"
 	"net/http"
-	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/philippta/flyscrape"
@@ -18,77 +17,55 @@ import (
 )
 
 func TestHeaders(t *testing.T) {
-	testCases := []struct {
-		name        string
-		headersFn   func() headers.Module
-		wantHeaders map[string][]string
-	}{
-		{
-			name: "empty custom headers",
-			headersFn: func() headers.Module {
-				return headers.Module{
-					Headers: map[string]string{},
-				}
-			},
-			wantHeaders: map[string][]string{"User-Agent": {"flyscrape/0.1"}},
-		},
-		{
-			name: "no duplicate headers between default and custom",
-			headersFn: func() headers.Module {
-				return headers.Module{
-					Headers: map[string]string{
-						"Authorization": "Basic ZGVtbzpwQDU1dzByZA==",
-					},
-				}
-			},
-			wantHeaders: map[string][]string{
-				"Authorization": {"Basic ZGVtbzpwQDU1dzByZA=="},
-				"User-Agent":    {"flyscrape/0.1"},
+	gotHeaders := map[string]string{}
+	sentHeaders := map[string]string{
+		"Authorization": "Basic ZGVtbzpwQDU1dzByZA==",
+		"User-Agent":    "Gecko/1.0",
+	}
+
+	mods := []flyscrape.Module{
+		hook.Module{
+			AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper {
+				return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
+					for k := range r.Header {
+						gotHeaders[k] = r.Header.Get(k)
+					}
+					return flyscrape.MockResponse(200, "")
+				})
 			},
 		},
-		{
-			name: "duplicate headers between default and custom",
-			headersFn: func() headers.Module {
-				return headers.Module{
-					Headers: map[string]string{
-						"Authorization": "Basic ZGVtbzpwQDU1dzByZA==",
-						"User-Agent":    "Gecko/1.0",
-					},
-				}
-			},
-			wantHeaders: map[string][]string{
-				"Authorization": {"Basic ZGVtbzpwQDU1dzByZA=="},
-				"User-Agent":    {"Gecko/1.0"},
-			},
+		&starturl.Module{URL: "http://www.example.com"},
+		&headers.Module{
+			Headers: sentHeaders,
 		},
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			var headers map[string][]string
+	scraper := flyscrape.NewScraper()
+	scraper.Modules = mods
+	scraper.Run()
 
-			mods := []flyscrape.Module{
-				&starturl.Module{URL: "http://www.example.com"},
-				hook.Module{
-					AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper {
-						return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
-							headers = r.Header
-							return rt.RoundTrip(r)
-						})
-					},
-				},
-				tc.headersFn(),
-			}
-
-			scraper := flyscrape.NewScraper()
-			scraper.Modules = mods
-			scraper.Run()
+	require.Equal(t, sentHeaders, gotHeaders)
+}
 
-			require.Truef(
-				t,
-				reflect.DeepEqual(tc.wantHeaders, headers),
-				fmt.Sprintf("expected: %v; actual: %v", tc.wantHeaders, headers),
-			)
-		})
+func TestHeadersRandomUserAgent(t *testing.T) {
+	var userAgent string
+	mods := []flyscrape.Module{
+		hook.Module{
+			AdaptTransportFn: func(rt http.RoundTripper) http.RoundTripper {
+				return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
+					userAgent = r.Header.Get("User-Agent")
+					return flyscrape.MockResponse(200, "")
+				})
+			},
+		},
+		&starturl.Module{URL: "http://www.example.com"},
+		&headers.Module{},
 	}
+
+	scraper := flyscrape.NewScraper()
+	scraper.Modules = mods
+	scraper.Run()
+
+	require.NotEmpty(t, userAgent)
+	require.True(t, strings.HasPrefix(userAgent, "Mozilla/5.0 ("))
 }
diff --git a/modules/headers/versions.go b/modules/headers/versions.go
new file mode 100644
index 0000000..eae19f6
--- /dev/null
+++ b/modules/headers/versions.go
@@ -0,0 +1,96 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package headers
+
+import (
+	_ "embed"
+	"fmt"
+	"math/rand"
+	"strings"
+)
+
+//go:generate bash -c "flyscrape run ../../examples/useragents/chrome.js | jq -r '[.[].data] | flatten | .[]' | sort -nr | uniq > versions_chrome.txt"
+//go:generate bash -c "flyscrape run ../../examples/useragents/firefox.js | jq -r '[.[].data] | flatten | .[]' | sort -nr | uniq > versions_firefox.txt"
+//go:generate bash -c "flyscrape run ../../examples/useragents/edge.js | jq -r '[.[].data] | flatten | .[]' | sort -nr | uniq > versions_edge.txt"
+//go:generate bash -c "flyscrape run ../../examples/useragents/opera.js | jq -r '[.[].data] | flatten | .[]' | sort -nr | uniq > versions_opera.txt"
+
+//go:embed versions_chrome.txt
+var versionsChromeRaw string
+var versionsChrome = strings.Split("\n", strings.TrimSpace(versionsChromeRaw))
+
+//go:embed versions_firefox.txt
+var versionsFirefoxRaw string
+var versionsFirefox = strings.Split("\n", strings.TrimSpace(versionsFirefoxRaw))
+
+//go:embed versions_edge.txt
+var versionsEdgeRaw string
+var versionsEdge = strings.Split("\n", strings.TrimSpace(versionsEdgeRaw))
+
+//go:embed versions_opera.txt
+var versionsOperaRaw string
+var versionsOpera = strings.Split("\n", strings.TrimSpace(versionsOperaRaw))
+
+//go:embed versions_macos.txt
+var versionsMacOSRaw string
+var versionsMacOS = strings.Split("\n", strings.TrimSpace(versionsMacOSRaw))
+
+//go:embed versions_windows.txt
+var versionsWindowsRaw string
+var versionsWindows = strings.Split("\n", strings.TrimSpace(versionsWindowsRaw))
+
+//go:embed versions_linux.txt
+var versionsLinuxRaw string
+var versionsLinux = strings.Split("\n", strings.TrimSpace(versionsLinuxRaw))
+
+func randomUAChrome() string {
+	f := "Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36"
+	return fmt.Sprintf(f, randomOS(), random(versionsChrome))
+}
+
+func randomUAFirefox() string {
+	f := "Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s"
+	ver := random(versionsFirefox)
+	return fmt.Sprintf(f, randomOS(), ver, ver)
+}
+
+func randomUAEdge() string {
+	f := "Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/%s"
+	return fmt.Sprintf(f, randomOS(), random(versionsEdge))
+}
+
+func randomUAOpera() string {
+	f := "Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/"
+	return fmt.Sprintf(f, randomOS(), random(versionsOpera))
+}
+
+func randomUserAgent() string {
+	switch rand.Intn(4) {
+	case 0:
+		return randomUAChrome()
+	case 1:
+		return randomUAFirefox()
+	case 2:
+		return randomUAEdge()
+	case 3:
+		return randomUAOpera()
+	}
+	panic("rand.Intn is broken")
+}
+
+func randomOS() string {
+	switch rand.Intn(3) {
+	case 0:
+		return random(versionsMacOS)
+	case 1:
+		return random(versionsWindows)
+	case 2:
+		return random(versionsLinux)
+	}
+	panic("rand.Intn is broken")
+}
+
+func random(ss []string) string {
+	return ss[rand.Intn(len(ss))]
+}
diff --git a/modules/headers/versions_chrome.txt b/modules/headers/versions_chrome.txt
new file mode 100644
index 0000000..b15a432
--- /dev/null
+++ b/modules/headers/versions_chrome.txt
@@ -0,0 +1,342 @@
+123.0.6312.52
+123.0.6312.40
+122.0.6261.94
+122.0.6261.90
+122.0.6261.89
+122.0.6261.69
+122.0.6261.64
+122.0.6261.62
+122.0.6261.51
+122.0.6261.48
+122.0.6261.129
+122.0.6261.128
+122.0.6261.119
+122.0.6261.112
+122.0.6261.111
+122.0.6261.105
+122.0.6045.214
+121.0.6167.85
+121.0.6167.66
+121.0.6167.184
+121.0.6167.178
+121.0.6167.171
+121.0.6167.164
+121.0.6167.160
+121.0.6167.143
+121.0.6167.139
+121.0.6167.138
+121.0.6167.101
+120.0.6099.71
+120.0.6099.50
+120.0.6099.43
+120.0.6099.230
+120.0.6099.216
+120.0.6099.210
+120.0.6099.199
+120.0.6099.193
+120.0.6099.130
+120.0.6099.129
+120.0.6099.119
+120.0.6099.109
+120.0.6099.101
+119.0.6045.66
+119.0.6045.41
+119.0.6045.214
+119.0.6045.200
+119.0.6045.199
+119.0.6045.193
+119.0.6045.169
+119.0.6045.163
+119.0.6045.134
+119.0.6045.124
+119.0.6045.123
+119.0.6045.109
+118.0.5993.96
+118.0.5993.92
+118.0.5993.89
+118.0.5993.88
+118.0.5993.80
+118.0.5993.71
+118.0.5993.70
+118.0.5993.69
+118.0.5993.65
+118.0.5993.58
+118.0.5993.118
+118.0.5993.117
+118.0.5993.111
+117.0.5938.92
+117.0.5938.89
+117.0.5938.88
+117.0.5938.82
+117.0.5938.60
+117.0.5938.153
+117.0.5938.150
+117.0.5938.149
+117.0.5938.140
+117.0.5938.117
+117.0.5938.108
+117.0.5938.104
+116.0.5845.97
+116.0.5845.96
+116.0.5845.92
+116.0.5845.90
+116.0.5845.188
+116.0.5845.187
+116.0.5845.180
+116.0.5845.179
+116.0.5845.177
+116.0.5845.172
+116.0.5845.163
+116.0.5845.146
+116.0.5845.141
+116.0.5845.140
+116.0.5845.118
+116.0.5845.114
+116.0.5845.111
+116.0.5845.110
+116.0.5845.103
+115.0.5790.84
+115.0.5790.166
+115.0.5790.160
+115.0.5790.138
+115.0.5790.136
+115.0.5790.130
+115.0.5790.114
+115.0.5790.110
+115.0.5790.102
+114.0.5735.99
+114.0.5735.61
+114.0.5735.60
+114.0.5735.58
+114.0.5735.57
+114.0.5735.50
+114.0.5735.198
+114.0.5735.196
+114.0.5735.133
+114.0.5735.131
+114.0.5735.130
+114.0.5735.124
+114.0.5735.106
+113.0.5672.92
+113.0.5672.77
+113.0.5672.76
+113.0.5672.69
+113.0.5672.163
+113.0.5672.162
+113.0.5672.134
+113.0.5672.126
+113.0.5672.121
+113.0.5672.109
+112.0.5615.70
+112.0.5615.69
+112.0.5615.48
+112.0.5615.47
+112.0.5615.46
+112.0.5615.167
+112.0.5615.165
+112.0.5615.137
+112.0.5615.136
+112.0.5615.135
+112.0.5615.101
+112.0.5615.100
+111.0.5563.72
+111.0.5563.58
+111.0.5563.57
+111.0.5563.54
+111.0.5563.147
+111.0.5563.116
+111.0.5563.115
+111.0.5563.110
+111.0.5563.101
+110.0.5481.96
+110.0.5481.83
+110.0.5481.65
+110.0.5481.64
+110.0.5481.63
+110.0.5481.177
+110.0.5481.154
+110.0.5481.153
+110.0.5481.114
+110.0.5481.104
+110.0.5481.100
+109.0.5414.94
+109.0.5414.83
+109.0.5414.165
+109.0.5414.149
+109.0.5414.141
+109.0.5414.125
+109.0.5414.119
+109.0.5414.112
+108.0.5359.98
+108.0.5359.94
+108.0.5359.61
+108.0.5359.52
+108.0.5359.128
+108.0.5359.124
+108.0.5359.112
+107.0.5304.91
+107.0.5304.87
+107.0.5304.66
+107.0.5304.54
+107.0.5304.141
+107.0.5304.121
+107.0.5304.110
+107.0.5304.105
+107.0.5304.101
+106.0.5249.92
+106.0.5249.91
+106.0.5249.75
+106.0.5249.70
+106.0.5249.65
+106.0.5249.60
+106.0.5249.126
+106.0.5249.119
+106.0.5249.118
+106.0.5249.112
+106.0.5249.103
+105.0.5195.98
+105.0.5195.79
+105.0.5195.77
+105.0.5195.69
+105.0.5195.68
+105.0.5195.147
+105.0.5195.136
+105.0.5195.134
+105.0.5195.129
+105.0.5195.125
+105.0.5195.124
+105.0.5195.112
+105.0.5195.102
+105.0.5195.100
+104.0.5112.99
+104.0.5112.97
+104.0.5112.88
+104.0.5112.71
+104.0.5112.69
+104.0.5112.102
+104.0.5112.101
+103.0.5060.71
+103.0.5060.70
+103.0.5060.66
+103.0.5060.64
+103.0.5060.63
+103.0.5060.54
+103.0.5060.53
+103.0.5060.134
+103.0.5060.132
+103.0.5060.129
+103.0.5060.114
+102.0.5005.99
+102.0.5005.78
+102.0.5005.67
+102.0.5005.59
+102.0.5005.115
+101.0.4951.67
+101.0.4951.64
+101.0.4951.61
+101.0.4951.54
+101.0.4951.44
+101.0.4951.41
+100.0.4896.88
+100.0.4896.85
+100.0.4896.79
+100.0.4896.77
+100.0.4896.75
+100.0.4896.58
+100.0.4896.56
+100.0.4896.127
+99.0.4844.88
+99.0.4844.84
+99.0.4844.83
+99.0.4844.82
+99.0.4844.74
+99.0.4844.73
+99.0.4844.59
+99.0.4844.48
+99.0.4844.47
+98.0.4758.97
+98.0.4758.87
+98.0.4758.85
+98.0.4758.109
+98.0.4758.102
+98.0.4758.101
+97.0.4692.99
+97.0.4692.84
+97.0.4692.72
+97.0.4664.98
+97.0.4664.87
+97.0.4664.70
+96.0.4664.94
+96.0.4664.93
+96.0.4664.92
+96.0.4664.53
+96.0.4664.45
+96.0.4664.36
+96.0.4664.116
+96.0.4664.110
+96.0.4664.104
+96.0.4664.101
+95.0.4638.69
+95.0.4638.50
+94.0.4606.85
+94.0.4606.81
+94.0.4606.80
+94.0.4606.76
+94.0.4606.71
+94.0.4606.61
+94.0.4606.52
+94.0.4606.50
+93.0.4577.82
+93.0.4577.78
+93.0.4577.62
+93.0.4577.39
+92.0.4515.90
+92.0.4515.166
+92.0.4515.159
+92.0.4515.131
+92.0.4515.115
+92.0.4515.105
+91.0.4472.88
+91.0.4472.80
+91.0.4472.77
+91.0.4472.164
+91.0.4472.124
+91.0.4472.123
+91.0.4472.120
+91.0.4472.114
+91.0.4472.106
+91.0.4472.101
+90.0.4430.93
+90.0.4430.91
+90.0.4430.85
+90.0.4430.82
+90.0.4430.78
+90.0.4430.66
+90.0.4430.216
+90.0.4430.212
+90.0.4430.210
+89.0.4389.90
+89.0.4389.86
+89.0.4389.82
+89.0.4389.72
+89.0.4389.128
+89.0.4389.114
+89.0.4389.105
+88.0.4324.93
+88.0.4324.192
+88.0.4324.190
+88.0.4324.182
+88.0.4324.181
+88.0.4324.155
+88.0.4324.150
+88.0.4324.146
+88.0.4324.141
+87.0.4280.88
+87.0.4280.86
+87.0.4280.77
+87.0.4280.67
+87.0.4280.66
+87.0.4280.60
+87.0.4280.163
+87.0.4280.141
+87.0.4280.101
diff --git a/modules/headers/versions_edge.txt b/modules/headers/versions_edge.txt
new file mode 100644
index 0000000..1eda77a
--- /dev/null
+++ b/modules/headers/versions_edge.txt
@@ -0,0 +1,89 @@
+132.0.2957.11
+131.0.2903.9
+131.0.2903.48
+130.0.2849.5
+130.0.2849.46
+129.0.2792.52
+129.0.2792.12
+128.0.2739.5
+128.0.2739.42
+127.0.2651.8
+127.0.2651.74
+126.0.2592.56
+126.0.2592.13
+125.0.2535.51
+125.0.2535.13
+124.0.2478.51
+124.0.2478.10
+123.0.2420.53
+123.0.2420.10
+122.0.2365.8
+122.0.2365.52
+121.0.2277.83
+121.0.2277.4
+120.0.2210.7
+120.0.2210.61
+119.0.2151.44
+119.0.2151.12
+118.0.2088.46
+118.0.2088.11
+117.0.2045.9
+117.0.2045.31
+116.0.1938.54
+116.0.1938.29
+115.0.1901.7
+115.0.1901.183
+114.0.1823.37
+114.0.1823.11
+113.0.1774.9
+113.0.1774.3
+112.0.1722.34
+112.0.1722.11
+111.0.1661.41
+111.0.1661.15
+110.0.1587.41
+110.0.1587.17
+109.0.1518.49
+109.0.1518.14
+108.0.1462.42
+108.0.1462.15
+107.0.1418.8
+107.0.1418.24
+106.0.1370.34
+106.0.1370.15
+105.0.1343.7
+105.0.1343.25
+104.0.1293.47
+104.0.1293.14
+103.0.1264.37
+103.0.1264.13
+102.0.1245.7
+102.0.1245.30
+101.0.1210.32
+101.0.1210.10
+100.0.1185.29
+100.0.1185.10
+99.0.1150.30
+99.0.1150.11
+98.0.1108.43
+98.0.1108.23
+97.0.1072.55
+97.0.1072.21
+96.0.1054.8
+96.0.1054.29
+95.0.1020.9
+95.0.1020.30
+94.0.992.9
+94.0.992.31
+93.0.961.38
+93.0.961.11
+92.0.902.9
+92.0.902.55
+91.0.864.37
+91.0.864.11
+90.0.818.8
+90.0.818.39
+89.0.774.48
+89.0.774.18
+88.0.705.50
+88.0.705.18
diff --git a/modules/headers/versions_firefox.txt b/modules/headers/versions_firefox.txt
new file mode 100644
index 0000000..ceccad8
--- /dev/null
+++ b/modules/headers/versions_firefox.txt
@@ -0,0 +1,309 @@
+132.0.2
+132.0.1
+132.0
+131.0.3
+131.0.2
+131.0
+130.0.1
+130.0
+129.0.2
+129.0.1
+129.0
+128.4.0
+128.3.1
+128.3.0
+128.2.0
+128.1.0
+128.0.3
+128.0.2
+128.0
+127.0.2
+127.0.1
+127.0
+126.0.1
+126.0
+125.0.3
+125.0.2
+125.0.1
+124.0.2
+124.0.1
+124.0
+123.0.1
+123.0
+122.0.1
+122.0
+121.0.1
+121.0
+120.0.1
+120.0
+119.0.1
+119.0
+118.0.2
+118.0.1
+118.0
+117.0.1
+117.0
+116.0.3
+116.0.2
+116.0.1
+116.0
+115.9.1
+115.9.0
+115.8.0
+115.7.0
+115.6.0
+115.5.0
+115.4.0
+115.3.1
+115.3.0
+115.2.1
+115.2.0
+115.17.0
+115.16.1
+115.16.0
+115.15.0
+115.14.0
+115.13.0
+115.12.0
+115.11.0
+115.10.0
+115.1.0
+115.0.3
+115.0.2
+115.0.1
+115.0
+114.0.2
+114.0.1
+114.0
+113.0.2
+113.0.1
+113.0
+112.0.2
+112.0.1
+112.0
+111.0.1
+111.0
+110.0.1
+110.0
+109.0.1
+109.0
+108.0.2
+108.0.1
+108.0
+107.0.1
+107.0
+106.0.5
+106.0.4
+106.0.3
+106.0.2
+106.0.1
+106.0
+105.0.3
+105.0.2
+105.0.1
+105.0
+104.0.2
+104.0.1
+104.0
+103.0.2
+103.0.1
+103.0
+102.9.0
+102.8.0
+102.7.0
+102.6.0
+102.5.0
+102.4.0
+102.3.0
+102.2.0
+102.15.1
+102.15.0
+102.14.0
+102.13.0
+102.12.0
+102.11.0
+102.10.0
+102.1.0
+102.0.1
+102.0
+101.0.1
+101.0
+100.0.2
+100.0.1
+100.0
+99.0.1
+99.0
+98.0.2
+98.0.1
+98.0
+97.0.2
+97.0.1
+97.0
+96.0.3
+96.0.2
+96.0.1
+96.0
+95.0.2
+95.0.1
+95.0
+94.0.2
+94.0.1
+94.0
+93.0
+92.0.1
+92.0
+91.9.1
+91.9.0
+91.8.0
+91.7.1
+91.7.0
+91.6.1
+91.6.0
+91.5.1
+91.5.0
+91.4.1
+91.4.0
+91.3.0
+91.2.0
+91.13.0
+91.12.0
+91.11.0
+91.10.0
+91.1.0
+91.0.2
+91.0.1
+91.0
+90.0.2
+90.0.1
+90.0
+89.0.2
+89.0.1
+89.0
+88.0.1
+88.0
+87.0
+86.0.1
+86.0
+85.0.2
+85.0.1
+85.0
+84.0.2
+84.0.1
+84.0
+83.0
+82.0.3
+82.0.2
+82.0.1
+82.0
+81.0.2
+81.0.1
+81.0
+80.0.1
+80.0
+79.0
+78.9.0
+78.8.0
+78.7.1
+78.7.0
+78.6.1
+78.6.0
+78.5.0
+78.4.1
+78.4.0
+78.3.1
+78.3.0
+78.2.0
+78.15.0
+78.14.0
+78.13.0
+78.12.0
+78.11.0
+78.10.1
+78.10.0
+78.1.0
+78.0.2
+78.0.1
+78.0
+77.0.1
+77.0
+76.0.1
+76.0
+75.0
+74.0.1
+74.0
+73.0.1
+73.0
+72.0.2
+72.0.1
+72.0
+71.0
+70.0.1
+70.0
+69.0.3
+69.0.2
+69.0.1
+69.0
+68.9.0
+68.8.0
+68.7.0
+68.6.1
+68.6.0
+68.5.0
+68.4.2
+68.4.1
+68.4.0
+68.3.0
+68.2.0
+68.12.0
+68.11.0
+68.10.0
+68.1.0
+68.0.2
+68.0.1
+68.0
+67.0.4
+67.0.3
+67.0.2
+67.0.1
+67.0
+66.0.5
+66.0.4
+66.0.3
+66.0.2
+66.0.1
+66.0
+65.0.2
+65.0.1
+65.0
+64.0.2
+64.0
+63.0.3
+63.0.1
+63.0
+62.0.3
+62.0.2
+62.0
+61.0.2
+61.0.1
+61.0
+60.9.0
+60.8.0
+60.7.2
+60.7.1
+60.7.0
+60.6.3
+60.6.2
+60.6.1
+60.6.0
+60.5.2
+60.5.1
+60.5.0
+60.4.0
+60.3.0
+60.2.2
+60.2.1
+60.2.0
+60.1.0
+60.0.2
+60.0.1
+60.0
diff --git a/modules/headers/versions_linux.txt b/modules/headers/versions_linux.txt
new file mode 100644
index 0000000..eb836ee
--- /dev/null
+++ b/modules/headers/versions_linux.txt
@@ -0,0 +1,2 @@
+X11; Linux x86_64
+X11; Ubuntu; Linux x86_64
diff --git a/modules/headers/versions_macos.txt b/modules/headers/versions_macos.txt
new file mode 100644
index 0000000..2439a62
--- /dev/null
+++ b/modules/headers/versions_macos.txt
@@ -0,0 +1,87 @@
+Macintosh; Intel Mac OS X 11_2_1
+Macintosh; Intel Mac OS X 11_2_2
+Macintosh; Intel Mac OS X 11_2_3
+Macintosh; Intel Mac OS X 11_3_1
+Macintosh; Intel Mac OS X 11_3_5
+Macintosh; Intel Mac OS X 11_4_3
+Macintosh; Intel Mac OS X 11_5_1
+Macintosh; Intel Mac OS X 11_5_2
+Macintosh; Intel Mac OS X 11_6_1
+Macintosh; Intel Mac OS X 11_6_2
+Macintosh; Intel Mac OS X 11_6_3
+Macintosh; Intel Mac OS X 11_6_4
+Macintosh; Intel Mac OS X 11_6_5
+Macintosh; Intel Mac OS X 11_6_6
+Macintosh; Intel Mac OS X 11_6_7
+Macintosh; Intel Mac OS X 11_6_8
+Macintosh; Intel Mac OS X 11_7
+Macintosh; Intel Mac OS X 11_7_1
+Macintosh; Intel Mac OS X 11_7_10
+Macintosh; Intel Mac OS X 11_7_2
+Macintosh; Intel Mac OS X 11_7_3
+Macintosh; Intel Mac OS X 11_7_4
+Macintosh; Intel Mac OS X 11_7_5
+Macintosh; Intel Mac OS X 11_7_6
+Macintosh; Intel Mac OS X 11_7_7
+Macintosh; Intel Mac OS X 11_7_8
+Macintosh; Intel Mac OS X 11_7_9
+Macintosh; Intel Mac OS X 12_0_1
+Macintosh; Intel Mac OS X 12_1_2
+Macintosh; Intel Mac OS X 12_2_1
+Macintosh; Intel Mac OS X 12_3_1
+Macintosh; Intel Mac OS X 12_3_4
+Macintosh; Intel Mac OS X 12_4_2
+Macintosh; Intel Mac OS X 12_5_1
+Macintosh; Intel Mac OS X 12_5_3
+Macintosh; Intel Mac OS X 12_6_1
+Macintosh; Intel Mac OS X 12_6_2
+Macintosh; Intel Mac OS X 12_6_3
+Macintosh; Intel Mac OS X 12_6_4
+Macintosh; Intel Mac OS X 12_6_5
+Macintosh; Intel Mac OS X 12_6_6
+Macintosh; Intel Mac OS X 12_6_7
+Macintosh; Intel Mac OS X 12_6_8
+Macintosh; Intel Mac OS X 12_6_9
+Macintosh; Intel Mac OS X 12_7
+Macintosh; Intel Mac OS X 12_7_1
+Macintosh; Intel Mac OS X 12_7_2
+Macintosh; Intel Mac OS X 12_7_3
+Macintosh; Intel Mac OS X 12_7_4
+Macintosh; Intel Mac OS X 12_7_5
+Macintosh; Intel Mac OS X 12_7_6
+Macintosh; Intel Mac OS X 13_0_1
+Macintosh; Intel Mac OS X 13_0_5
+Macintosh; Intel Mac OS X 13_1_1
+Macintosh; Intel Mac OS X 13_2_1
+Macintosh; Intel Mac OS X 13_3_1
+Macintosh; Intel Mac OS X 13_3_3
+Macintosh; Intel Mac OS X 13_4_1
+Macintosh; Intel Mac OS X 13_5_1
+Macintosh; Intel Mac OS X 13_5_2
+Macintosh; Intel Mac OS X 13_6
+Macintosh; Intel Mac OS X 13_6_1
+Macintosh; Intel Mac OS X 13_6_2
+Macintosh; Intel Mac OS X 13_6_3
+Macintosh; Intel Mac OS X 13_6_4
+Macintosh; Intel Mac OS X 13_6_5
+Macintosh; Intel Mac OS X 13_6_6
+Macintosh; Intel Mac OS X 13_6_7
+Macintosh; Intel Mac OS X 13_6_8
+Macintosh; Intel Mac OS X 13_6_9
+Macintosh; Intel Mac OS X 13_7
+Macintosh; Intel Mac OS X 13_7_1
+Macintosh; Intel Mac OS X 14_0_6
+Macintosh; Intel Mac OS X 14_1_1
+Macintosh; Intel Mac OS X 14_1_2
+Macintosh; Intel Mac OS X 14_2_1
+Macintosh; Intel Mac OS X 14_2_2
+Macintosh; Intel Mac OS X 14_3_1
+Macintosh; Intel Mac OS X 14_4_1
+Macintosh; Intel Mac OS X 14_4_3
+Macintosh; Intel Mac OS X 14_5_2
+Macintosh; Intel Mac OS X 14_6_1
+Macintosh; Intel Mac OS X 14_6_2
+Macintosh; Intel Mac OS X 14_7
+Macintosh; Intel Mac OS X 14_7_1
+Macintosh; Intel Mac OS X 15_0
+Macintosh; Intel Mac OS X 15_0_1
diff --git a/modules/headers/versions_opera.txt b/modules/headers/versions_opera.txt
new file mode 100644
index 0000000..8f7dd3f
--- /dev/null
+++ b/modules/headers/versions_opera.txt
@@ -0,0 +1,675 @@
+110.0.5130.82
+110.0.5130.8
+110.0.5130.66
+110.0.5130.49
+110.0.5130.4
+110.0.5130.39
+110.0.5130.35
+110.0.5130.23
+110.0.5130.17
+110.0.5130.13
+110.0.5117.0
+110.0.5111.0
+110.0.5104.0
+109.0.5097.80
+109.0.5097.59
+109.0.5097.5
+109.0.5097.45
+109.0.5097.38
+109.0.5097.33
+109.0.5097.24
+109.0.5097.0
+109.0.5089.0
+109.0.5083.0
+109.0.5076.0
+109.0.5069.0
+108.0.5067.40
+108.0.5067.24
+108.0.5067.20
+108.0.5067.14
+108.0.5067.10
+108.0.5063.0
+108.0.5054.0
+108.0.5047.0
+107.0.5045.8
+107.0.5045.4
+107.0.5045.36
+107.0.5045.21
+107.0.5045.15
+107.0.5045.11
+107.0.5041.0
+107.0.5035.0
+107.0.5019.0
+107.0.5012.0
+107.0.5004.0
+106.0.4971.0
+105.0.4970.6
+105.0.4970.48
+105.0.4970.34
+105.0.4970.29
+105.0.4970.21
+105.0.4970.16
+105.0.4970.13
+105.0.4970.10
+105.0.4963.0
+105.0.4957.0
+105.0.4950.0
+104.0.4944.54
+104.0.4944.36
+104.0.4944.33
+104.0.4944.3
+104.0.4944.28
+104.0.4944.23
+104.0.4944.18
+104.0.4944.10
+104.0.4941.0
+104.0.4934.0
+103.0.4928.34
+103.0.4928.3
+103.0.4928.26
+103.0.4928.16
+103.0.4928.0
+103.0.4920.0
+103.0.4906.0
+103.0.4899.0
+103.0.4892.0
+103.0.4885.0
+102.0.4880.78
+102.0.4880.70
+102.0.4880.6
+102.0.4880.56
+102.0.4880.51
+102.0.4880.46
+102.0.4880.40
+102.0.4880.38
+102.0.4880.33
+102.0.4880.28
+102.0.4880.16
+102.0.4880.10
+102.0.4879.0
+102.0.4871.0
+102.0.4864.0
+102.0.4857.0
+102.0.4850.0
+101.0.4843.58
+101.0.4843.5
+101.0.4843.43
+101.0.4843.33
+101.0.4843.25
+101.0.4843.19 –
+101.0.4843.13
+101.0.4843.10
+101.0.4843.0
+101.0.4836.0
+101.0.4829.0
+101.0.4822.0
+100.0.4815.76
+100.0.4815.54
+100.0.4815.47
+100.0.4815.30
+100.0.4815.2
+100.0.4815.13
+100.0.4815.0
+100.0.4809.0
+100.0.4801.0
+100.0.4796.0
+100.0.4790.0
+99.0.4788.9
+99.0.4788.88
+99.0.4788.77
+99.0.4788.65
+99.0.4788.5
+99.0.4788.47
+99.0.4788.31
+99.0.4788.13
+99.0.4780.0
+99.0.4765.0
+98.0.4759.6
+98.0.4759.39
+98.0.4759.3
+98.0.4759.21
+98.0.4759.15
+98.0.4759.1
+98.0.4756.0
+98.0.4746.0
+98.0.4739.0
+98.0.4732.0
+98.0.4725.0
+97.0.4719.83
+97.0.4719.63
+97.0.4719.43
+97.0.4719.4
+97.0.4719.28
+97.0.4719.26
+97.0.4719.17
+97.0.4719.11
+97.0.4718.0
+97.0.4711.0
+97.0.4704.0
+97.0.4697.0
+96.0.4693.80
+96.0.4693.50
+96.0.4693.31
+96.0.4693.20
+96.0.4693.16
+96.0.4693.12
+96.0.4691.0
+96.0.4674.0
+96.0.4660.0
+96.0.4653.0
+96.0.4640.0
+95.0.4635.84
+95.0.4635.46
+95.0.4635.37
+95.0.4635.28
+95.0.4635.25
+95.0.4635.20
+95.0.4635.15
+95.0.4635.12
+95.0.4635.10
+95.0.4632.0
+95.0.4625.0
+95.0.4618.0
+95.0.4612.0
+94.0.4606.8
+94.0.4606.76
+94.0.4606.65
+94.0.4606.54
+94.0.4606.38
+94.0.4606.26
+94.0.4606.19
+94.0.4606.14
+94.0.4604.0
+94.0.4597.0
+93.0.4585.70
+93.0.4585.7
+93.0.4585.64
+93.0.4585.39
+93.0.4585.37
+93.0.4585.3
+93.0.4585.21
+93.0.4585.11
+93.0.4582.0
+93.0.4575.0
+93.0.4569.0
+92.0.4561.8
+92.0.4561.43
+92.0.4561.33
+92.0.4561.30
+92.0.4561.21
+92.0.4561.11
+92.0.4561.0
+92.0.4555.0
+92.0.4547.0
+92.0.4540.0
+92.0.4526.0
+92.0.4519.0
+91.0.4516.9
+91.0.4516.77
+91.0.4516.65
+91.0.4516.6
+91.0.4516.3
+91.0.4516.20
+91.0.4516.16
+91.0.4514.0
+91.0.4505.0
+91.0.4498.0
+91.0.4491.0
+91.0.4484.0
+90.0.4480.84
+90.0.4480.80
+90.0.4480.54
+90.0.4480.48
+90.0.4480.41
+90.0.4480.37
+90.0.4480.30
+90.0.4480.25
+90.0.4480.107
+90.0.4477.0
+90.0.4470.0
+90.0.4463.0
+90.0.4457.0
+90.0.4450.0
+89.0.4447.83
+89.0.4447.71
+89.0.4447.51
+89.0.4447.48
+89.0.4447.38
+89.0.4447.37
+89.0.4447.33
+89.0.4447.31
+89.0.4447.20
+89.0.4447.12
+89.0.4447.101
+89.0.4443.0
+89.0.4436.0
+89.0.4428.0
+89.0.4422.0
+89.0.4415.0
+88.0.4412.74
+88.0.4412.53
+88.0.4412.40
+88.0.4412.27
+88.0.4412.20
+88.0.4412.18
+88.0.4412.13
+88.0.4401.0
+88.0.4395.0
+87.0.4390.8
+87.0.4390.45
+87.0.4390.36
+87.0.4390.25
+87.0.4390.21
+87.0.4390.17
+87.0.4388.0
+87.0.4382.0
+87.0.4374.0
+87.0.4366.0
+86.0.4363.9
+86.0.4363.59
+86.0.4363.50
+86.0.4363.32
+86.0.4363.22
+86.0.4363.17
+86.0.4363.15
+86.0.4363.12
+86.0.4359.0
+86.0.4351.0
+86.0.4344.0
+85.0.4341.75
+85.0.4341.60
+85.0.4341.6
+85.0.4341.47
+85.0.4341.28
+85.0.4341.18
+85.0.4341.13
+85.0.4341.10
+85.0.4338.0
+85.0.4331.0
+85.0.4323.0
+84.0.4316.9
+84.0.4316.42
+84.0.4316.31
+84.0.4316.21
+84.0.4316.14
+84.0.4316.0
+84.0.4309.0
+84.0.4302.0
+84.0.4295.0
+84.0.4284.0
+84.0.4274.0
+84.0.4267.0
+84.0.4260.0
+83.0.4254.9
+83.0.4254.62
+83.0.4254.54
+83.0.4254.5
+83.0.4254.27
+83.0.4254.19
+83.0.4254.16
+83.0.4254.14
+83.0.4253.0
+83.0.4246.0
+83.0.4239.0
+83.0.4232.0
+82.0.4227.7
+82.0.4227.58
+82.0.4227.43
+82.0.4227.4
+82.0.4227.33
+82.0.4227.23
+82.0.4227.13
+82.0.4226.0
+82.0.4218.0
+82.0.4210.0
+82.0.4203.0
+81.0.4196.60
+81.0.4196.54
+81.0.4196.37
+81.0.4196.31
+81.0.4196.27
+81.0.4196.14
+81.0.4196.11
+81.0.4196.0
+81.0.4189.0
+81.0.4183.0
+81.0.4175.0
+80.0.4170.72
+80.0.4170.7
+80.0.4170.63
+80.0.4170.40
+80.0.4170.4
+80.0.4170.16
+80.0.4170.11
+80.0.4170.0
+80.0.4162.0
+80.0.4157.0
+80.0.4150.0
+79.0.4143.72
+79.0.4143.50
+79.0.4143.3
+79.0.4143.22
+79.0.4143.15
+79.0.4142.0
+79.0.4135.0
+79.0.4128.0
+79.0.4114.0
+79.0.4105.0
+79.0.4100.0
+78.0.4093.79
+78.0.4093.68
+78.0.4093.46
+78.0.4093.34
+78.0.4093.184
+78.0.4093.147
+78.0.4093.112
+78.0.4093.103
+78.0.4093.0
+78.0.4086.0
+78.0.4079.0
+78.0.4072.0
+78.0.4066.0
+78.0.4058.0
+77.0.4054.91
+77.0.4054.90
+77.0.4054.80
+77.0.4054.64
+77.0.4054.38
+77.0.4054.277 –
+77.0.4054.254 –
+77.0.4054.203 –
+77.0.4054.19
+77.0.4054.172 –
+77.0.4054.146 –
+77.0.4054.14
+77.0.4051.0
+77.0.4046.0
+77.0.4039.0
+77.0.4032.0
+77.0.4028.0
+77.0.4023.0
+76.0.4017.94
+76.0.4017.88
+76.0.4017.59
+76.0.4017.5
+76.0.4017.40
+76.0.4017.177
+76.0.4017.154
+76.0.4017.137
+76.0.4017.123
+76.0.4017.107
+76.0.4009.0
+76.0.3995.0
+76.0.3989.0
+76.0.3981.0
+76.0.3974.0
+75.0.3969.93
+75.0.3969.60
+75.0.3969.50
+75.0.3969.35
+75.0.3969.218
+75.0.3969.171
+75.0.3969.141
+75.0.3969.14
+75.0.3967.0
+75.0.3960.0
+75.0.3953.0
+75.0.3946.0
+75.0.3939.0
+75.0.3932.0
+75.0.3925.0
+74.0.3911.75
+74.0.3911.63
+74.0.3911.42
+74.0.3911.232
+74.0.3911.22
+74.0.3911.218
+74.0.3911.203
+74.0.3911.160
+74.0.3911.107
+74.0.3904.0
+74.0.3897.0
+74.0.3890.0
+74.0.3883.0
+74.0.3876.0
+74.0.3870.0
+74.0.3862.0
+73.0.3856.344
+73.0.3856.329
+73.0.3856.31
+73.0.3856.284
+73.0.3856.257
+73.0.3856.235
+73.0.3856.208
+73.0.3856.184
+73.0.3856.156
+73.0.3856.0
+73.0.3847.0
+73.0.3841.0
+73.0.3834.0
+73.0.3827.0
+73.0.3820.0
+72.0.3815.86
+72.0.3815.49
+72.0.3815.400
+72.0.3815.378
+72.0.3815.320
+72.0.3815.200
+72.0.3815.186
+72.0.3815.148
+72.0.3815.133
+72.0.3814.0
+72.0.3807.0
+72.0.3798.0
+72.0.3791.0
+72.0.3784.0
+72.0.3779.0
+71.0.3770.97
+71.0.3770.81
+71.0.3770.50
+71.0.3770.271
+71.0.3770.228
+71.0.3770.198
+71.0.3770.148
+71.0.3770.126
+71.0.3770.0
+71.0.3763.0
+71.0.3756.0
+71.0.3749.0
+71.0.3742.0
+71.0.3735.0 –
+70.0.3728.95
+70.0.3728.8
+70.0.3728.71
+70.0.3728.59
+70.0.3728.46
+70.0.3728.21
+70.0.3728.189
+70.0.3728.144
+70.0.3728.133
+70.0.3728.119
+70.0.3728.106
+70.0.3728.0
+70.0.3721.0
+70.0.3714.0
+70.0.3707.0
+70.0.3701.0
+70.0.3693.0
+69.0.3686.95
+69.0.3686.77
+69.0.3686.7
+69.0.3686.57
+69.0.3686.49
+69.0.3686.36
+69.0.3686.30
+69.0.3686.21
+69.0.3686.2
+69.0.3686.12
+69.0.3686.0
+69.0.3679.0
+69.0.3673.0
+69.0.3665.0
+69.0.3660.0
+69.0.3653.0
+69.0.3651.0
+69.0.3645.0
+69.0.3638.0
+69.0.3630.0
+69.0.3623.0
+68.0.3618.91
+68.0.3618.63
+68.0.3618.56
+68.0.3618.5
+68.0.3618.45/68.0.3618.46
+68.0.3618.41
+68.0.3618.36
+68.0.3618.31
+68.0.3618.3
+68.0.3618.24
+68.0.3618.18
+68.0.3618.173
+68.0.3618.165
+68.0.3618.104
+68.0.3616.0
+68.0.3609.0
+68.0.3602.0
+68.0.3590.0
+68.0.3581.0
+67.0.3575.97
+67.0.3575.8
+67.0.3575.79
+67.0.3575.53
+67.0.3575.31
+67.0.3575.28
+67.0.3575.23
+67.0.3575.2
+67.0.3575.137
+67.0.3575.13
+67.0.3575.115
+67.0.3574.0
+67.0.3564.0
+67.0.3554.0
+67.0.3541.0
+67.0.3536.0
+67.0.3523.0
+66.0.3515.72
+66.0.3515.7
+66.0.3515.44
+66.0.3515.36
+66.0.3515.3
+66.0.3515.27
+66.0.3515.21
+66.0.3515.2
+66.0.3515.14
+66.0.3515.103
+66.0.3511.0
+66.0.3508.0
+66.0.3502.0
+66.0.3494.0
+66.0.3487.0
+66.0.3480.0
+66.0.3475.0
+66.0.3472.0
+65.0.3467.78
+65.0.3467.72
+65.0.3467.7
+65.0.3467.69
+65.0.3467.62
+65.0.3467.48
+65.0.3467.38
+65.0.3467.32
+65.0.3467.24
+65.0.3467.16
+65.0.3466.0
+65.0.3459.0
+65.0.3454.0
+65.0.3450.0
+65.0.3445.0
+65.0.3437.0
+65.0.3430.0
+65.0.3425.0
+64.0.3417.92
+64.0.3417.83
+64.0.3417.8
+64.0.3417.73
+64.0.3417.61
+64.0.3417.54
+64.0.3417.47
+64.0.3417.41
+64.0.3417.32
+64.0.3417.19
+64.0.3417.119
+64.0.3417.11
+64.0.3416.0
+64.0.3409.0
+64.0.3407.0
+64.0.3401.0
+64.0.3396.0
+64.0.3394.0
+64.0.3388.0
+64.0.3380.0
+64.0.3372.0
+63.0.3368.94
+63.0.3368.88
+63.0.3368.8
+63.0.3368.66
+63.0.3368.53
+63.0.3368.51
+63.0.3368.43
+63.0.3368.35
+63.0.3368.33
+63.0.3368.29
+63.0.3368.22
+63.0.3368.17
+63.0.3368.14
+63.0.3367.0
+63.0.3359.0
+63.0.3353.0
+63.0.3349.0
+63.0.3347.0
+62.0.3331.99
+62.0.3331.8
+62.0.3331.66
+62.0.3331.55
+62.0.3331.5
+62.0.3331.43
+62.0.3331.2
+62.0.3331.18
+62.0.3331.14
+62.0.3331.119
+62.0.3331.116
+62.0.3331.10
+62.0.3323.0
+62.0.3319.0
+61.0.3298.6
+61.0.3298.3
+61.0.3296.0
+61.0.3290.0
+61.0.3282.0
+61.0.3275.0
+61.0.3271.0
+61.0.3268.0
+60.0.3255.95
+60.0.3255.84
+60.0.3255.83
+60.0.3255.8
+60.0.3255.79
+60.0.3255.70
+60.0.3255.60
+60.0.3255.59
+60.0.3255.57
+60.0.3255.56
+60.0.3255.4
+60.0.3255.37
+60.0.3255.27
+60.0.3255.20
+60.0.3255.170
+60.0.3255.151
+60.0.3255.15
+60.0.3255.116
+60.0.3255.109
+60.0.3255.103
+60.0.3254.0
+60.0.3248.0
+60.0.3242.0
+60.0.3236.0
diff --git a/modules/headers/versions_windows.txt b/modules/headers/versions_windows.txt
new file mode 100644
index 0000000..9fbc0aa
--- /dev/null
+++ b/modules/headers/versions_windows.txt
@@ -0,0 +1,2 @@
+Windows NT 10.0; Win64; x64
+Windows NT 10.0; WOW64