diff options
| author | Philipp Tanlak <philipp.tanlak@gmail.com> | 2024-11-24 23:16:31 +0100 |
|---|---|---|
| committer | Philipp Tanlak <philipp.tanlak@gmail.com> | 2024-11-24 23:16:31 +0100 |
| commit | 175cb0700b86134d3b04eab23d100dce2c3b6d56 (patch) | |
| tree | a48b69aafeaf4a8690fba9b3d0bbe4ac9e64ecf3 | |
| parent | 653be4548b0883532665997a4624e56fdf7c02a6 (diff) | |
Change cache backend from sqlite to bbolt
| -rw-r--r-- | .github/workflows/release.yaml | 30 | ||||
| -rw-r--r-- | .goreleaser.yaml | 102 | ||||
| -rw-r--r-- | go.mod | 25 | ||||
| -rw-r--r-- | go.sum | 43 | ||||
| -rw-r--r-- | modules/cache/boltstore.go | 74 | ||||
| -rw-r--r-- | modules/cache/boltstore_test.go (renamed from modules/cache/sqlitestore_test.go) | 6 | ||||
| -rw-r--r-- | modules/cache/cache.go | 7 | ||||
| -rw-r--r-- | modules/cache/memstore.go | 13 | ||||
| -rw-r--r-- | modules/cache/sqlitestore.go | 60 | ||||
| -rw-r--r-- | modules/cookies/cookies.go | 4 |
10 files changed, 127 insertions, 237 deletions
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 19b0e7c..9ca9888 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,11 +8,6 @@ on: permissions: contents: write -env: - GO_VERSION: '1.21.4' - PKG_NAME: 'github.com/philippta/flyscrape' - DOCKER_IMG: ghcr.io/goreleaser/goreleaser-cross - jobs: release: name: Release @@ -25,16 +20,15 @@ jobs: - name: Fetch Git tags run: git fetch --force --tags - - name: Build and Release - run: | - docker run \ - --rm \ - --privileged \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(pwd)/sysroot:/sysroot \ - -v $(pwd):/go/src/${PKG_NAME} \ - -w /go/src/${PKG_NAME} \ - -e CGO_ENABLED=1 \ - -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ - ghcr.io/goreleaser/goreleaser-cross:v${GO_VERSION} \ - release --clean + - name: Set up Go + uses: actions/setup-go@v5.1.0 + with: + go-version: '1.23.3' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6.1.0 + with: + version: '~> v2' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index da630a7..43f3c3c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -4,118 +4,22 @@ before: - go test ./... builds: - # Linux amd64 - - id: linux-amd64 + - id: flyscrape main: ./cmd/flyscrape env: - - CGO_ENABLED=1 - - CC=x86_64-linux-gnu-gcc - - CXX=x86_64-linux-gnu-g++ + - CGO_ENABLED=0 ldflags: - -s -w + - -extldflags "-static" - -X github.com/philippta/flyscrape.Version={{.Tag}} flags: - -mod=readonly tags: - osusergo - netgo - - sqlite_omit_load_extension - goos: - - linux - goarch: - - amd64 - - # Linux arm64 - - id: linux-arm64 - main: ./cmd/flyscrape - env: - - CGO_ENABLED=1 - - CC=aarch64-linux-gnu-gcc - - CXX=aarch64-linux-gnu-g++ - ldflags: - - -s -w - - -X github.com/philippta/flyscrape.Version={{.Tag}} - flags: - - -mod=readonly - tags: - - osusergo - - netgo - - sqlite_omit_load_extension - goos: - - linux - goarch: - - arm64 - goarm: - - 7 - - # MacOS amd64 - - id: macos-amd64 - main: ./cmd/flyscrape - env: - - CGO_ENABLED=1 - - CC=o64-clang - - CXX=o64-clang++ - ldflags: - - -s -w - - -X github.com/philippta/flyscrape.Version={{.Tag}} - flags: - - -mod=readonly - tags: - - osusergo - - netgo - - sqlite_omit_load_extension - goos: - - darwin - goarch: - - amd64 - - # MacOS arm64 - - id: macos-arm64 - main: ./cmd/flyscrape - env: - - CGO_ENABLED=1 - - CC=oa64-clang - - CXX=oa64-clang++ - ldflags: - - -s -w - - -X github.com/philippta/flyscrape.Version={{.Tag}} - flags: - - -mod=readonly - tags: - - osusergo - - netgo - - sqlite_omit_load_extension - goos: - - darwin - goarch: - - arm64 - - # Window amd64 - - id: windows-amd64 - main: ./cmd/flyscrape - env: - - CGO_ENABLED=1 - - CC=x86_64-w64-mingw32-gcc - - CXX=x86_64-w64-mingw32-g++ - ldflags: - - -s -w - flags: - - -mod=readonly - tags: - - osusergo - - netgo - - sqlite_omit_load_extension - goos: - - windows - goarch: - - amd64 archives: - format: tar.gz - name_template: >- - {{ .ProjectName }}_ - {{- if eq .Os "darwin" }}macos{{else}}{{ .Os }}{{ end }}_ - {{- .Arch }} format_overrides: - goos: windows format: zip @@ -1,10 +1,12 @@ module github.com/philippta/flyscrape -go 1.21 +go 1.23 + +toolchain go1.23.3 require ( github.com/PuerkitoBio/goquery v1.8.1 - github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200 + github.com/browserutils/kooky v0.2.2 github.com/cornelk/hashmap v1.0.8 github.com/dop251/goja v0.0.0-20230919151941-fc55792775de github.com/dop251/goja_nodejs v0.0.0-20230914102007-198ba9a8b098 @@ -12,11 +14,11 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/go-rod/rod v0.114.7 github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 - github.com/mattn/go-sqlite3 v1.14.17 github.com/nlnwa/whatwg-url v0.4.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/sjson v1.2.5 - golang.org/x/sync v0.5.0 + go.etcd.io/bbolt v1.3.11 + golang.org/x/sync v0.9.0 ) require ( @@ -25,7 +27,6 @@ require ( github.com/Velocidex/yaml/v2 v2.2.8 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/bits-and-blooms/bitset v1.5.0 // indirect - github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/go-ini/ini v1.67.0 // indirect @@ -34,7 +35,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gonuts/binary v0.2.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect - github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100 // indirect + github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect @@ -46,12 +47,12 @@ require ( github.com/ysmood/got v0.34.1 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect - github.com/zalando/go-keyring v0.2.3 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/zalando/go-keyring v0.2.5 // indirect + golang.org/x/crypto v0.29.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect www.velocidex.com/golang/go-ese v0.2.0 // indirect ) @@ -18,10 +18,8 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ= -github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck= -github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200 h1:GDuK/WZUQhqvOQpZvK74iE+YdwfUVppCaLohrq6k90M= -github.com/browserutils/kooky v0.2.1-0.20240119192416-d4f81abd0200/go.mod h1:lCmEKO6kWHgukZblrl7p2Po8R6VWLcX6N5pKAb3Ar9M= +github.com/browserutils/kooky v0.2.2 h1:uLKlE294eXudGEAt/NjOrL5Nzbi57ZtkuWwKZ1hT13I= +github.com/browserutils/kooky v0.2.2/go.mod h1:Ls7BAtUgrzzi5AfD1T4CqDu7mhHAaGMwCx6kH2nnjHI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -63,8 +61,8 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8q github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM= -github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100 h1:rG3VnJUnAWyiv7qYmmdOdSapzz6HM+zb9/uRFr0T5EM= -github.com/keybase/go-keychain v0.0.0-20230523030712-b5615109f100/go.mod h1:qDHUvIjGZJUtdPtuP4WMu5/U4aVWbFw1MhlkJqCGmCQ= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -75,8 +73,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/nlnwa/whatwg-url v0.4.0 h1:B3kFb5EL7KILeBkhrlQvFi41Ex0p4ropVA9brt5ungI= github.com/nlnwa/whatwg-url v0.4.0/go.mod h1:pLzpJjFPtA+n7RCLvp0GBxvDHa/2ckNCBK9mfEeNOMQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -125,15 +121,16 @@ github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= -github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +github.com/zalando/go-keyring v0.2.5 h1:Bc2HHpjALryKD62ppdEzaFG6VxL6Bc+5v0LYpN8Lba8= +github.com/zalando/go-keyring v0.2.5/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -141,13 +138,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -161,14 +158,14 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -176,8 +173,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/modules/cache/boltstore.go b/modules/cache/boltstore.go new file mode 100644 index 0000000..7176445 --- /dev/null +++ b/modules/cache/boltstore.go @@ -0,0 +1,74 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cache + +import ( + "errors" + "log" + "os" + + "go.etcd.io/bbolt" +) + +var cache = []byte("cache") + +func NewBoltStore(file string) *BoltStore { + db, err := bbolt.Open(file, 0644, nil) + if err != nil { + log.Printf("cache: failed to create database file %q: %v\n", file, err) + os.Exit(1) + } + + c := &BoltStore{db: db} + + return c +} + +type BoltStore struct { + db *bbolt.DB +} + +func (s *BoltStore) Get(key string) ([]byte, bool) { + var value []byte + + err := s.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(cache) + if bucket == nil { + return errors.New("bucket not found") + } + + v := bucket.Get([]byte(key)) + if v == nil { + return errors.New("key not found") + } + + value = make([]byte, len(v)) + copy(value, v) + + return nil + }) + if err != nil { + return nil, false + } + return value, true +} + +func (s *BoltStore) Set(key string, value []byte) { + err := s.db.Update(func(tx *bbolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(cache) + if err != nil { + return err + } + + return bucket.Put([]byte(key), value) + }) + if err != nil { + log.Printf("cache: failed to insert cache key %q: %v\n", key, err) + } +} + +func (s *BoltStore) Close() { + s.db.Close() +} diff --git a/modules/cache/sqlitestore_test.go b/modules/cache/boltstore_test.go index 769a69f..0af954f 100644 --- a/modules/cache/sqlitestore_test.go +++ b/modules/cache/boltstore_test.go @@ -12,12 +12,12 @@ import ( "github.com/stretchr/testify/require" ) -func TestSQLiteStore(t *testing.T) { - dir, err := os.MkdirTemp("", "sqlitestore") +func TestBoltStore(t *testing.T) { + dir, err := os.MkdirTemp("", "boltstore") require.NoError(t, err) defer os.RemoveAll(dir) - store := cache.NewSQLiteStore(dir + "/test.db") + store := cache.NewBoltStore(dir + "/test.db") v, ok := store.Get("foo") require.Nil(t, v) diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 401aa49..7164506 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -34,15 +34,12 @@ func (Module) ModuleInfo() flyscrape.ModuleInfo { func (m *Module) Provision(ctx flyscrape.Context) { switch { - case m.Cache == "memory": - m.store = NewMemStore() - case m.Cache == "file": file := replaceExt(ctx.ScriptName(), ".cache") - m.store = NewSQLiteStore(file) + m.store = NewBoltStore(file) case strings.HasPrefix(m.Cache, "file:"): - m.store = NewSQLiteStore(strings.TrimPrefix(m.Cache, "file:")) + m.store = NewBoltStore(strings.TrimPrefix(m.Cache, "file:")) } } diff --git a/modules/cache/memstore.go b/modules/cache/memstore.go deleted file mode 100644 index 0f4d9e2..0000000 --- a/modules/cache/memstore.go +++ /dev/null @@ -1,13 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cache - -import ( - "github.com/cornelk/hashmap" -) - -func NewMemStore() *hashmap.Map[string, []byte] { - return hashmap.New[string, []byte]() -} diff --git a/modules/cache/sqlitestore.go b/modules/cache/sqlitestore.go deleted file mode 100644 index 778699b..0000000 --- a/modules/cache/sqlitestore.go +++ /dev/null @@ -1,60 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cache - -import ( - "database/sql" - "fmt" - "log" - "os" - - _ "github.com/mattn/go-sqlite3" -) - -func NewSQLiteStore(file string) *SQLiteStore { - db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?_timeout=5000&_journal=WAL", file)) - if err != nil { - log.Printf("cache: failed to create database file %q: %v\n", file, err) - os.Exit(1) - } - - c := &SQLiteStore{db: db} - c.migrate() - - return c -} - -type SQLiteStore struct { - db *sql.DB -} - -func (s *SQLiteStore) Get(key string) ([]byte, bool) { - var value []byte - if err := s.db.QueryRow(`SELECT value FROM cache WHERE key = ? LIMIT 1`, key).Scan(&value); err != nil { - return nil, false - } - return value, true -} - -func (s *SQLiteStore) Set(key string, value []byte) { - if _, err := s.db.Exec(`INSERT INTO cache (key, value) VALUES (?, ?)`, key, value); err != nil { - log.Printf("cache: failed to insert cache key %q: %v\n", key, err) - } -} - -func (s *SQLiteStore) Close() { - s.db.Close() -} - -func (s *SQLiteStore) migrate() { - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS cache (key TEXT, value BLOB)`); err != nil { - log.Printf("cache: failed to create cache table: %v\n", err) - os.Exit(1) - } - if _, err := s.db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS cache_key_idx ON cache(key)`); err != nil { - log.Printf("cache: failed to create cache index: %v\n", err) - os.Exit(1) - } -} diff --git a/modules/cookies/cookies.go b/modules/cookies/cookies.go index 2af2a27..2f57a3f 100644 --- a/modules/cookies/cookies.go +++ b/modules/cookies/cookies.go @@ -51,10 +51,6 @@ func (m Module) AdaptTransport(t http.RoundTripper) http.RoundTripper { return flyscrape.RoundTripFunc(func(r *http.Request) (*http.Response, error) { for _, store := range stores { for _, cookie := range store.Cookies(r.URL) { - // Unquote cookie value - if len(cookie.Value) >= 2 && cookie.Value[0] == '"' && cookie.Value[len(cookie.Value)-1] == '"' { - cookie.Value = cookie.Value[1 : len(cookie.Value)-2] - } r.AddCookie(cookie) } } |