chromedp/browser.go

357 lines
8.5 KiB
Go
Raw Normal View History

// Package chromedp is a high level Chrome DevTools Protocol client that
// simplifies driving browsers for scraping, unit testing, or profiling web
// pages using the CDP.
//
// chromedp requires no third-party dependencies, implementing the async Chrome
// DevTools Protocol entirely in Go.
package chromedp
import (
"context"
"encoding/json"
"log"
"os"
"sync/atomic"
"github.com/mailru/easyjson"
"github.com/chromedp/cdproto"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/cdproto/target"
)
// Browser is the high-level Chrome DevTools Protocol browser manager, handling
// the browser process runner, WebSocket clients, associated targets, and
// network, page, and DOM events.
type Browser struct {
conn Transport
// next is the next message id.
next int64
// tabQueue is the queue used to create new target handlers, once a new
// tab is created and attached to. The newly created Target is sent back
// via tabResult.
tabQueue chan newTab
tabResult chan *Target
// cmdQueue is the outgoing command queue.
cmdQueue chan cmdJob
// logging funcs
logf func(string, ...interface{})
errf func(string, ...interface{})
// The optional fields below are helpful for some tests.
// process can be initialized by the allocators which start a process
// when allocating a browser.
process *os.Process
// userDataDir can be initialized by the allocators which set up user
// data dirs directly.
userDataDir string
}
type newTab struct {
targetID target.ID
sessionID target.SessionID
}
type cmdJob struct {
msg *cdproto.Message
resp chan *cdproto.Message
}
// NewBrowser creates a new browser.
func NewBrowser(ctx context.Context, urlstr string, opts ...BrowserOption) (*Browser, error) {
conn, err := DialContext(ctx, ForceIP(urlstr))
if err != nil {
return nil, err
}
b := &Browser{
conn: conn,
tabQueue: make(chan newTab, 1),
tabResult: make(chan *Target, 1),
cmdQueue: make(chan cmdJob),
logf: log.Printf,
}
// apply options
for _, o := range opts {
if err := o(b); err != nil {
return nil, err
}
}
// ensure errf is set
if b.errf == nil {
b.errf = func(s string, v ...interface{}) { b.logf("ERROR: "+s, v...) }
}
go b.run(ctx)
return b, nil
}
// Shutdown shuts down the browser.
func (b *Browser) Shutdown() error {
if b.conn != nil {
if err := b.send(cdproto.CommandBrowserClose, nil); err != nil {
b.errf("could not close browser: %v", err)
}
return b.conn.Close()
}
return nil
}
// send writes the supplied message and params.
func (b *Browser) send(method cdproto.MethodType, params easyjson.RawMessage) error {
msg := &cdproto.Message{
ID: atomic.AddInt64(&b.next, 1),
Method: method,
Params: params,
}
return b.conn.Write(msg)
}
func (b *Browser) newExecutorForTarget(ctx context.Context, targetID target.ID, sessionID target.SessionID) *Target {
if targetID == "" {
panic("empty target ID")
}
if sessionID == "" {
panic("empty session ID")
}
b.tabQueue <- newTab{targetID, sessionID}
return <-b.tabResult
}
func (b *Browser) Execute(ctx context.Context, method string, params json.Marshaler, res json.Unmarshaler) error {
paramsMsg := emptyObj
if params != nil {
var err error
if paramsMsg, err = json.Marshal(params); err != nil {
return err
}
}
id := atomic.AddInt64(&b.next, 1)
ch := make(chan *cdproto.Message, 1)
b.cmdQueue <- cmdJob{
msg: &cdproto.Message{
ID: id,
Method: cdproto.MethodType(method),
Params: paramsMsg,
},
resp: ch,
}
select {
case msg := <-ch:
switch {
case msg == nil:
return ErrChannelClosed
case msg.Error != nil:
return msg.Error
case res != nil:
return json.Unmarshal(msg.Result, res)
}
case <-ctx.Done():
return ctx.Err()
}
return nil
}
type tabEvent struct {
sessionID target.SessionID
msg *cdproto.Message
}
func (b *Browser) run(ctx context.Context) {
defer b.conn.Close()
cancel := FromContext(ctx).cancel
// tabEventQueue is the queue of incoming target events, to be routed by
// their session ID.
tabEventQueue := make(chan tabEvent, 1)
// resQueue is the incoming command result queue.
resQueue := make(chan *cdproto.Message, 1)
// This goroutine continuously reads events from the websocket
// connection. The separate goroutine is needed since a websocket read
// is blocking, so it cannot be used in a select statement.
go func() {
for {
msg, err := b.conn.Read()
if err != nil {
// If the websocket failed, most likely Chrome
// was closed or crashed. Cancel the entire
// Browser context to stop all activity.
cancel()
return
}
if msg.Method == cdproto.EventRuntimeExceptionThrown {
ev := new(runtime.EventExceptionThrown)
if err := json.Unmarshal(msg.Params, ev); err != nil {
b.errf("%s", err)
continue
}
b.errf("%+v\n", ev.ExceptionDetails)
continue
}
var sessionID target.SessionID
if msg.Method == cdproto.EventTargetReceivedMessageFromTarget {
event := new(target.EventReceivedMessageFromTarget)
if err := json.Unmarshal(msg.Params, event); err != nil {
b.errf("%s", err)
continue
}
sessionID = event.SessionID
msg = new(cdproto.Message)
if err := json.Unmarshal([]byte(event.Message), msg); err != nil {
b.errf("%s", err)
continue
}
}
switch {
case msg.Method != "":
if sessionID == "" {
// TODO: are we interested in browser events?
continue
}
tabEventQueue <- tabEvent{
sessionID: sessionID,
msg: msg,
}
case msg.ID != 0:
// We can't process the response here, as it's
// another goroutine that maintans respByID.
resQueue <- msg
default:
b.errf("ignoring malformed incoming message (missing id or method): %#v", msg)
}
}
}()
// This goroutine handles tabs, as well as routing events to each tab
// via the pages map.
go func() {
// This map is only safe for use within this goroutine, so don't
// declare it as a Browser field.
pages := make(map[target.SessionID]*Target, 1024)
for {
select {
case tab := <-b.tabQueue:
if _, ok := pages[tab.sessionID]; ok {
b.errf("executor for %q already exists", tab.sessionID)
}
t := &Target{
browser: b,
TargetID: tab.targetID,
SessionID: tab.sessionID,
eventQueue: make(chan *cdproto.Message, 1024),
waitQueue: make(chan func(cur *cdp.Frame) bool, 1024),
frames: make(map[cdp.FrameID]*cdp.Frame),
logf: b.logf,
errf: b.errf,
}
go t.run(ctx)
pages[tab.sessionID] = t
b.tabResult <- t
case event := <-tabEventQueue:
page, ok := pages[event.sessionID]
if !ok {
b.errf("unknown session ID %q", event.sessionID)
continue
}
select {
case page.eventQueue <- event.msg:
default:
panic("eventQueue is full")
}
case <-ctx.Done():
return
}
}
}()
respByID := make(map[int64]chan *cdproto.Message)
// This goroutine handles sending commands to the browser, and sending
// responses back for each of these commands via respByID.
for {
select {
case res := <-resQueue:
resp, ok := respByID[res.ID]
if !ok {
b.errf("id %d not present in response map", res.ID)
continue
}
if resp != nil {
// resp could be nil, if we're not interested in
// this response; for CommandSendMessageToTarget.
resp <- res
close(resp)
}
delete(respByID, res.ID)
case q := <-b.cmdQueue:
if _, ok := respByID[q.msg.ID]; ok {
b.errf("id %d already present in response map", q.msg.ID)
continue
}
respByID[q.msg.ID] = q.resp
if q.msg.Method == "" {
// Only register the chananel in respByID;
// useful for CommandSendMessageToTarget.
continue
}
if err := b.conn.Write(q.msg); err != nil {
b.errf("%s", err)
continue
}
case <-ctx.Done():
return
}
}
}
// BrowserOption is a browser option.
type BrowserOption func(*Browser) error
// WithLogf is a browser option to specify a func to receive general logging.
func WithLogf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
b.logf = f
return nil
}
}
// WithErrorf is a browser option to specify a func to receive error logging.
func WithErrorf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
b.errf = f
return nil
}
}
// WithConsolef is a browser option to specify a func to receive chrome log events.
//
// Note: NOT YET IMPLEMENTED.
func WithConsolef(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
return nil
}
}