examples/flyscrapelib.go


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

package main

import "github.com/philippta/flyscrape/flyscrape"

var config = flyscrape.Config{
	URL:      "https://news.ycombinator.com/",
	Browser:  true,
	Headless: false,
}

func scrape(ctx flyscrape.Context) any {
	var (
		post         = ctx.Doc.Find(".athing.submission").First()
		title        = post.Find(".titleline > a").Text()
		commentsLink = post.Next().Find("a").Last().Attr("href")
		comments     = ctx.Scrape(commentsLink, scrapeComments)
	)

	return flyscrape.M{
		"title":    title,
		"comments": comments,
	}
}

func scrapeComments(ctx flyscrape.Context) any {
	var comments []flyscrape.M

	for _, comment := range ctx.Doc.Find(".comtr").All() {
		comments = append(comments, flyscrape.M{
			"author": comment.Find(".hnuser").Text(),
			"text":   comment.Find(".commtext").Text(),
		})

	}

	return comments
}

func main() {
	flyscrape.Run(config, scrape)
}