summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPhilipp Tanlak <philipp.tanlak@gmail.com>2024-11-24 23:16:31 +0100
committerPhilipp Tanlak <philipp.tanlak@gmail.com>2024-11-24 23:16:31 +0100
commit175cb0700b86134d3b04eab23d100dce2c3b6d56 (patch)
treea48b69aafeaf4a8690fba9b3d0bbe4ac9e64ecf3
parent653be4548b0883532665997a4624e56fdf7c02a6 (diff)
Change cache backend from sqlite to bbolt
-rw-r--r--.github/workflows/release.yaml30
-rw-r--r--.goreleaser.yaml102
-rw-r--r--go.mod25
-rw-r--r--go.sum43
-rw-r--r--modules/cache/boltstore.go74
-rw-r--r--modules/cache/boltstore_test.go (renamed from modules/cache/sqlitestore_test.go)6
-rw-r--r--modules/cache/cache.go7
-rw-r--r--modules/cache/memstore.go13
-rw-r--r--modules/cache/sqlitestore.go60
-rw-r--r--modules/cookies/cookies.go4
10 files changed, 127 insertions, 237 deletions
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 19b0e7c..9ca9888 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -8,11 +8,6 @@ on:
permissions:
contents: write
-env:
- GO_VERSION: '1.21.4'
- PKG_NAME: 'github.com/philippta/flyscrape'
- DOCKER_IMG: ghcr.io/goreleaser/goreleaser-cross
-
jobs:
release:
name: Release
@@ -25,16 +20,15 @@ jobs:
- name: Fetch Git tags
run: git fetch --force --tags
- - name: Build and Release
- run: |
- docker run \
- --rm \
- --privileged \
- -v /var/run/docker.sock:/var/run/docker.sock \
- -v $(pwd)/sysroot:/sysroot \
- -v $(pwd):/go/src/${PKG_NAME} \
- -w /go/src/${PKG_NAME} \
- -e CGO_ENABLED=1 \
- -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
- ghcr.io/goreleaser/goreleaser-cross:v${GO_VERSION} \
- release --clean
+ - name: Set up Go
+ uses: actions/setup-go@v5.1.0
+ with:
+ go-version: '1.23.3'
+
+ - name: Run GoReleaser
+ uses: goreleaser/goreleaser-action@v6.1.0
+ with:
+ version: '~> v2'
+ args: release --clean
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index da630a7..43f3c3c 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -4,118 +4,22 @@ before:
- go test ./...
builds:
- # Linux amd64
- - id: linux-amd64
+ - id: flyscrape
main: ./cmd/flyscrape
env:
- - CGO_ENABLED=1
- - CC=x86_64-linux-gnu-gcc
- - CXX=x86_64-linux-gnu-g++
+ - CGO_ENABLED=0
ldflags:
- -s -w
+ - -extldflags "-static"
- -X github.com/philippta/flyscrape.Version={{.Tag}}
flags:
- -mod=readonly
tags:
- osusergo
- netgo
- - sqlite_omit_load_extension
- goos:
- - linux
- goarch:
- - amd64
-
- # Linux arm64
- - id: linux-arm64
- main: ./cmd/flyscrape
- env:
- - CGO_ENABLED=1
- - CC=aarch64-linux-gnu-gcc
- - CXX=aarch64-linux-gnu-g++
- ldflags:
- - -s -w
- - -X github.com/philippta/flyscrape.Version={{.Tag}}
- flags:
- - -mod=readonly
- tags:
- - osusergo
- - netgo
- - sqlite_omit_load_extension
- goos:
- - linux
- goarch:
- - arm64
- goarm:
- - 7
-
- # MacOS amd64
- - id: macos-amd64
- main: ./cmd/flyscrape
- env:
- - CGO_ENABLED=1
- - CC=o64-clang
- - CXX=o64-clang++
- ldflags:
- - -s -w
- - -X github.com/philippta/flyscrape.Version={{.Tag}}
- flags:
- - -mod=readonly
- tags:
- - osusergo
- - netgo
- - sqlite_omit_load_extension
- goos:
- - darwin
- goarch:
- - amd64
-
- # MacOS arm64
- - id: macos-arm64
- main: ./cmd/flyscrape
- env:
- - CGO_ENABLED=1
- - CC=oa64-clang
- - CXX=oa64-clang++
- ldflags:
- - -s -w
- - -X github.com/philippta/flyscrape.Version={{.Tag}}
- flags:
- - -mod=readonly
- tags:
- - osusergo
- - netgo
- - sqlite_omit_load_extension
- goos:
- - darwin
- goarch:
- - arm64
-
- # Window amd64
- - id: windows-amd64
- main: ./cmd/flyscrape
- env:
- - CGO_ENABLED=1
- - CC=x86_64-w64-mingw32-gcc
- - CXX=x86_64-w64-mingw32-g++
- ldflags:
- - -s -w
- flags:
- - -mod=readonly
- tags:
- - osusergo
- - netgo
- - sqlite_omit_load_extension
- goos:
- - windows
- goarch:
- - amd64
archives:
- format: tar.gz
- name_template: >-
- {{ .ProjectName }}_
- {{- if eq .Os "darwin" }}macos{{else}}{{ .Os }}{{ end }}_
- {{- .Arch }}
format_overrides:
- goos: windows
format: zip
diff --git a/go.mod b/go.mod
index 4a2048c..838ad5f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,10 +1,12 @@
module github.com/philippta/flyscrape
-go 1.21
+go 1.23
+
+toolchain go1.23.3
require (
github.com/PuerkitoBio/goquery v1.8.1
- github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200
+ github.com/browserutils/kooky v0.2.2
github.com/cornelk/hashmap v1.0.8
github.com/dop251/goja v0.0.0-20230919151941-fc55792775de
github.com/dop251/goja_nodejs v0.0.0-20230914102007-198ba9a8b098
@@ -12,11 +14,11 @@ require (
github.com/fsnotify/fsnotify v1.6.0
github.com/go-rod/rod v0.114.7
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3
- github.com/mattn/go-sqlite3 v1.14.17
github.com/nlnwa/whatwg-url v0.4.0
github.com/stretchr/testify v1.8.4
github.com/tidwall/sjson v1.2.5
- golang.org/x/sync v0.5.0
+ go.etcd.io/bbolt v1.3.11
+ golang.org/x/sync v0.9.0
)
require (
@@ -25,7 +27,6 @@ require (
github.com/Velocidex/yaml/v2 v2.2.8 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/bits-and-blooms/bitset v1.5.0 // indirect
- github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
@@ -34,7 +35,7 @@ require (
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gonuts/binary v0.2.0 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
- github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100 // indirect
+ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
@@ -46,12 +47,12 @@ require (
github.com/ysmood/got v0.34.1 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.8.0 // indirect
- github.com/zalando/go-keyring v0.2.3 // indirect
- golang.org/x/crypto v0.19.0 // indirect
- golang.org/x/net v0.21.0 // indirect
- golang.org/x/sys v0.17.0 // indirect
- golang.org/x/term v0.17.0 // indirect
- golang.org/x/text v0.14.0 // indirect
+ github.com/zalando/go-keyring v0.2.5 // indirect
+ golang.org/x/crypto v0.29.0 // indirect
+ golang.org/x/net v0.31.0 // indirect
+ golang.org/x/sys v0.27.0 // indirect
+ golang.org/x/term v0.26.0 // indirect
+ golang.org/x/text v0.20.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
www.velocidex.com/golang/go-ese v0.2.0 // indirect
)
diff --git a/go.sum b/go.sum
index 19875e2..6e74da2 100644
--- a/go.sum
+++ b/go.sum
@@ -18,10 +18,8 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
-github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
-github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200 h1:GDuK/WZUQhqvOQpZvK74iE+YdwfUVppCaLohrq6k90M=
-github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200/go.mod h1:lCmEKO6kWHgukZblrl7p2Po8R6VWLcX6N5pKAb3Ar9M=
+github.com/browserutils/kooky v0.2.2 h1:uLKlE294eXudGEAt/NjOrL5Nzbi57ZtkuWwKZ1hT13I=
+github.com/browserutils/kooky v0.2.2/go.mod h1:Ls7BAtUgrzzi5AfD1T4CqDu7mhHAaGMwCx6kH2nnjHI=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -63,8 +61,8 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8q
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk=
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM=
-github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100 h1:rG3VnJUnAWyiv7qYmmdOdSapzz6HM+zb9/uRFr0T5EM=
-github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100/go.mod h1:qDHUvIjGZJUtdPtuP4WMu5/U4aVWbFw1MhlkJqCGmCQ=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
+github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -75,8 +73,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/nlnwa/whatwg-url v0.4.0 h1:B3kFb5EL7KILeBkhrlQvFi41Ex0p4ropVA9brt5ungI=
github.com/nlnwa/whatwg-url v0.4.0/go.mod h1:pLzpJjFPtA+n7RCLvp0GBxvDHa/2ckNCBK9mfEeNOMQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -125,15 +121,16 @@ github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R
github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms=
-github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk=
+github.com/zalando/go-keyring v0.2.5 h1:Bc2HHpjALryKD62ppdEzaFG6VxL6Bc+5v0LYpN8Lba8=
+github.com/zalando/go-keyring v0.2.5/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk=
+go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
+go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
+golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -141,13 +138,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -161,14 +158,14 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -176,8 +173,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
diff --git a/modules/cache/boltstore.go b/modules/cache/boltstore.go
new file mode 100644
index 0000000..7176445
--- /dev/null
+++ b/modules/cache/boltstore.go
@@ -0,0 +1,74 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package cache
+
+import (
+ "errors"
+ "log"
+ "os"
+
+ "go.etcd.io/bbolt"
+)
+
+var cache = []byte("cache")
+
+func NewBoltStore(file string) *BoltStore {
+ db, err := bbolt.Open(file, 0644, nil)
+ if err != nil {
+ log.Printf("cache: failed to create database file %q: %v\n", file, err)
+ os.Exit(1)
+ }
+
+ c := &BoltStore{db: db}
+
+ return c
+}
+
+type BoltStore struct {
+ db *bbolt.DB
+}
+
+func (s *BoltStore) Get(key string) ([]byte, bool) {
+ var value []byte
+
+ err := s.db.View(func(tx *bbolt.Tx) error {
+ bucket := tx.Bucket(cache)
+ if bucket == nil {
+ return errors.New("bucket not found")
+ }
+
+ v := bucket.Get([]byte(key))
+ if v == nil {
+ return errors.New("key not found")
+ }
+
+ value = make([]byte, len(v))
+ copy(value, v)
+
+ return nil
+ })
+ if err != nil {
+ return nil, false
+ }
+ return value, true
+}
+
+func (s *BoltStore) Set(key string, value []byte) {
+ err := s.db.Update(func(tx *bbolt.Tx) error {
+ bucket, err := tx.CreateBucketIfNotExists(cache)
+ if err != nil {
+ return err
+ }
+
+ return bucket.Put([]byte(key), value)
+ })
+ if err != nil {
+ log.Printf("cache: failed to insert cache key %q: %v\n", key, err)
+ }
+}
+
+func (s *BoltStore) Close() {
+ s.db.Close()
+}
diff --git a/modules/cache/sqlitestore_test.go b/modules/cache/boltstore_test.go
index 769a69f..0af954f 100644
--- a/modules/cache/sqlitestore_test.go
+++ b/modules/cache/boltstore_test.go
@@ -12,12 +12,12 @@ import (
"github.com/stretchr/testify/require"
)
-func TestSQLiteStore(t *testing.T) {
- dir, err := os.MkdirTemp("", "sqlitestore")
+func TestBoltStore(t *testing.T) {
+ dir, err := os.MkdirTemp("", "boltstore")
require.NoError(t, err)
defer os.RemoveAll(dir)
- store := cache.NewSQLiteStore(dir + "/test.db")
+ store := cache.NewBoltStore(dir + "/test.db")
v, ok := store.Get("foo")
require.Nil(t, v)
diff --git a/modules/cache/cache.go b/modules/cache/cache.go
index 401aa49..7164506 100644
--- a/modules/cache/cache.go
+++ b/modules/cache/cache.go
@@ -34,15 +34,12 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo {
func (m *Module) Provision(ctx flyscrape.Context) {
switch {
- case m.Cache == "memory":
- m.store = NewMemStore()
-
case m.Cache == "file":
file := replaceExt(ctx.ScriptName(), ".cache")
- m.store = NewSQLiteStore(file)
+ m.store = NewBoltStore(file)
case strings.HasPrefix(m.Cache, "file:"):
- m.store = NewSQLiteStore(strings.TrimPrefix(m.Cache, "file:"))
+ m.store = NewBoltStore(strings.TrimPrefix(m.Cache, "file:"))
}
}
diff --git a/modules/cache/memstore.go b/modules/cache/memstore.go
deleted file mode 100644
index 0f4d9e2..0000000
--- a/modules/cache/memstore.go
+++ /dev/null
@@ -1,13 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package cache
-
-import (
- "github.com/cornelk/hashmap"
-)
-
-func NewMemStore() *hashmap.Map[string, []byte] {
- return hashmap.New[string, []byte]()
-}
diff --git a/modules/cache/sqlitestore.go b/modules/cache/sqlitestore.go
deleted file mode 100644
index 778699b..0000000
--- a/modules/cache/sqlitestore.go
+++ /dev/null
@@ -1,60 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package cache
-
-import (
- "database/sql"
- "fmt"
- "log"
- "os"
-
- _ "github.com/mattn/go-sqlite3"
-)
-
-func NewSQLiteStore(file string) *SQLiteStore {
- db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?_timeout=5000&_journal=WAL", file))
- if err != nil {
- log.Printf("cache: failed to create database file %q: %v\n", file, err)
- os.Exit(1)
- }
-
- c := &SQLiteStore{db: db}
- c.migrate()
-
- return c
-}
-
-type SQLiteStore struct {
- db *sql.DB
-}
-
-func (s *SQLiteStore) Get(key string) ([]byte, bool) {
- var value []byte
- if err := s.db.QueryRow(`SELECT value FROM cache WHERE key = ? LIMIT 1`, key).Scan(&value); err != nil {
- return nil, false
- }
- return value, true
-}
-
-func (s *SQLiteStore) Set(key string, value []byte) {
- if _, err := s.db.Exec(`INSERT INTO cache (key, value) VALUES (?, ?)`, key, value); err != nil {
- log.Printf("cache: failed to insert cache key %q: %v\n", key, err)
- }
-}
-
-func (s *SQLiteStore) Close() {
- s.db.Close()
-}
-
-func (s *SQLiteStore) migrate() {
- if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS cache (key TEXT, value BLOB)`); err != nil {
- log.Printf("cache: failed to create cache table: %v\n", err)
- os.Exit(1)
- }
- if _, err := s.db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS cache_key_idx ON cache(key)`); err != nil {
- log.Printf("cache: failed to create cache index: %v\n", err)
- os.Exit(1)
- }
-}
diff --git a/modules/cookies/cookies.go b/modules/cookies/cookies.go
index 2af2a27..2f57a3f 100644
--- a/modules/cookies/cookies.go
+++ b/modules/cookies/cookies.go
@@ -51,10 +51,6 @@ func (m Module) AdaptTransport(t http.RoundTripper) http.RoundTripper {
return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) {
for _, store := range stores {
for _, cookie := range store.Cookies(r.URL) {
- // Unquote cookie value
- if len(cookie.Value) >= 2 && cookie.Value[0] == '"' && cookie.Value[len(cookie.Value)-1] == '"' {
- cookie.Value = cookie.Value[1 : len(cookie.Value)-2]
- }
r.AddCookie(cookie)
}
}