diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/output/json/json.go | 21 | ||||
| -rw-r--r-- | modules/output/ndjson/ndjson.go | 9 |
2 files changed, 22 insertions, 8 deletions
diff --git a/modules/output/json/json.go b/modules/output/json/json.go index 5b4e9d0..1937d17 100644 --- a/modules/output/json/json.go +++ b/modules/output/json/json.go @@ -11,6 +11,7 @@ import ( "io" "log" "os" + "sync" "time" "github.com/philippta/flyscrape" @@ -28,6 +29,7 @@ type Module struct { once bool w io.WriteCloser + mu *sync.Mutex } func (Module) ModuleInfo() flyscrape.ModuleInfo { @@ -42,6 +44,8 @@ func (m *Module) Provision(ctx flyscrape.Context) { return } + m.mu = &sync.Mutex{} + if m.Output.File == "" { m.w = nopCloser{os.Stdout} return @@ -64,13 +68,6 @@ func (m *Module) ReceiveResponse(resp *flyscrape.Response) { return } - if !m.once { - fmt.Fprintln(m.w, "[") - m.once = true - } else { - fmt.Fprintln(m.w, ",") - } - o := output{ URL: resp.Request.URL, Data: resp.Data, @@ -80,6 +77,16 @@ func (m *Module) ReceiveResponse(resp *flyscrape.Response) { o.Error = resp.Error.Error() } + m.mu.Lock() + defer m.mu.Unlock() + + if !m.once { + fmt.Fprintln(m.w, "[") + m.once = true + } else { + fmt.Fprintln(m.w, ",") + } + var buf bytes.Buffer enc := json.NewEncoder(&buf) enc.SetEscapeHTML(false) diff --git a/modules/output/ndjson/ndjson.go b/modules/output/ndjson/ndjson.go index 956b2ed..a24aaa6 100644 --- a/modules/output/ndjson/ndjson.go +++ b/modules/output/ndjson/ndjson.go @@ -9,6 +9,7 @@ import ( "io" "log" "os" + "sync" "time" "github.com/philippta/flyscrape" @@ -24,7 +25,8 @@ type Module struct { File string `json:"file"` } `json:"output"` - w io.WriteCloser + w io.WriteCloser + mu *sync.Mutex } func (Module) ModuleInfo() flyscrape.ModuleInfo { @@ -39,6 +41,8 @@ func (m *Module) Provision(ctx flyscrape.Context) { return } + m.mu = &sync.Mutex{} + if m.Output.File == "" { m.w = nopCloser{os.Stdout} return @@ -70,6 +74,9 @@ func (m *Module) ReceiveResponse(resp *flyscrape.Response) { o.Error = resp.Error.Error() } + m.mu.Lock() + defer m.mu.Unlock() + enc := json.NewEncoder(m.w) enc.SetEscapeHTML(false) enc.Encode(o) |