commons_go/chromedp
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: commons_go:master
Branches Tags
commons_go:master
commons_go:v0.1
commons_go:v0.1.4
commons_go:v0.1.3
commons_go:v0.1.2
commons_go:v0.1.1
commons_go:v0.1.0
..
compare: commons_go:v0.1.1
Branches Tags
commons_go:master
commons_go:v0.1
commons_go:v0.1.4
commons_go:v0.1.3
commons_go:v0.1.2
commons_go:v0.1.1
commons_go:v0.1.0

No commits in common. "master" and "v0.1.1" have entirely different histories.
master ... v0.1.1

45 changed files with 4033 additions and 2502 deletions
Show Stats Expand all files Collapse all files

15
.github/ISSUE_TEMPLATE vendored
View File

@ -1,15 +0,0 @@
#### What versions are you running?
<pre>
$ go list -m git.loafle.net/commons_go/chromedp
$ chromium --version
$ go version
</pre>
#### What did you do?
#### What did you expect to see?
#### What did you see instead?

13
.travis.yml
View File

@ -1,11 +1,14 @@
language: go
go:
- 1.12.x
- 1.10.x
- tip
addons:
apt:
chrome: stable
before_install:
- go get github.com/mattn/goveralls golang.org/x/vgo
script:
- go test -v ./...
- export CHROMEDP_TEST_RUNNER=google-chrome-stable
- export CHROMEDP_DISABLE_GPU=true
- vgo test -v -coverprofile=coverage.out
- goveralls -service=travis-ci -coverprofile=coverage.out

108
README.md
View File

@ -1,34 +1,118 @@
# About chromedp [![Build Status][1]][2] [![Coverage Status][3]][4]
Package chromedp is a faster, simpler way to drive browsers supporting the
[Chrome DevTools Protocol][5] in Go using the without external dependencies
(ie, Selenium, PhantomJS, etc).
Package chromedp is a faster, simpler way to drive browsers in Go using the
[Chrome Debugging Protocol][5] (for Chrome, Edge, Safari, etc) without external
dependencies (ie, Selenium, PhantomJS, etc).
**NOTE:** chromedp's API is currently unstable, and may change at a moments
notice. There are likely extremely bad bugs lurking in this code. **CAVEAT USER**.
## Installing
Install in the usual Go way:
Install in the usual way:
```sh
go get -u git.loafle.net/commons_go/chromedp
go get -u github.com/chromedp/chromedp
```
## Examples
## Using
Below is a simple Google search performed using chromedp (taken from
[examples/simple][6]):
This example shows logic for a simple search for a known website, clicking on
the right link, and then taking a screenshot of a specific element on the
loaded page and saving that to a local file on disk.
```go
// Command simple is a chromedp example demonstrating how to do a simple google
// search.
package main
import (
"context"
"fmt"
"io/ioutil"
"log"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp"
)
func main() {
var err error
// create context
ctxt, cancel := context.WithCancel(context.Background())
defer cancel()
// create chrome instance
c, err := chromedp.New(ctxt, chromedp.WithLog(log.Printf))
if err != nil {
log.Fatal(err)
}
// run task list
var site, res string
err = c.Run(ctxt, googleSearch("site:brank.as", "Home", &site, &res))
if err != nil {
log.Fatal(err)
}
// shutdown chrome
err = c.Shutdown(ctxt)
if err != nil {
log.Fatal(err)
}
// wait for chrome to finish
err = c.Wait()
if err != nil {
log.Fatal(err)
}
log.Printf("saved screenshot from search result listing `%s` (%s)", res, site)
}
func googleSearch(q, text string, site, res *string) chromedp.Tasks {
var buf []byte
sel := fmt.Sprintf(`//a[text()[contains(., '%s')]]`, text)
return chromedp.Tasks{
chromedp.Navigate(`https://www.google.com`),
chromedp.WaitVisible(`#hplogo`, chromedp.ByID),
chromedp.SendKeys(`#lst-ib`, q+"\n", chromedp.ByID),
chromedp.WaitVisible(`#res`, chromedp.ByID),
chromedp.Text(sel, res),
chromedp.Click(sel),
chromedp.WaitNotVisible(`.preloader-content`, chromedp.ByQuery),
chromedp.WaitVisible(`a[href*="twitter"]`, chromedp.ByQuery),
chromedp.Location(site),
chromedp.ScrollIntoView(`.banner-section.third-section`, chromedp.ByQuery),
chromedp.Sleep(2 * time.Second), // wait for animation to finish
chromedp.Screenshot(`.banner-section.third-section`, &buf, chromedp.ByQuery),
chromedp.ActionFunc(func(context.Context, cdp.Executor) error {
return ioutil.WriteFile("screenshot.png", buf, 0644)
}),
}
}
```
Please see the [examples][6] project for more examples. Please refer to the
[GoDoc API listing][7] for a summary of the API and Actions, which also contains
a few simple and runnable examples.
[GoDoc API listing][7] for a summary of the API and Actions.
## Resources
* [chromedp: A New Way to Drive the Web][8] - GopherCon SG 2017 talk
* [Chrome DevTools Protocol][5] - Chrome DevTools Protocol Domain documentation
* [Chrome DevTools Protocol][5] - Chrome Debugging Protocol Domain documentation
* [chromedp examples][6] - various `chromedp` examples
* [`github.com/chromedp/cdproto`][9] - GoDoc listing for the CDP domains used by `chromedp`
* [`github.com/chromedp/cdproto-gen`][10] - tool used to generate `cdproto`
* [`git.loafle.net/commons_go/chromedp-proxy`][11] - a simple CDP proxy for logging CDP clients and browsers
* [`github.com/chromedp/chromedp-proxy`][11] - a simple CDP proxy for logging/debugging CDP clients and browser instances
## TODO
* Move timeouts to context (defaults)
* Implement more query selector options (allow over riding context timeouts)
* Contextual actions for "dry run" (or via an accumulator?)
* Network loader / manager
@ -40,8 +124,8 @@ a few simple and runnable examples.
[4]: https://coveralls.io/github/chromedp/chromedp?branch=master
[5]: https://chromedevtools.github.io/devtools-protocol/
[6]: https://github.com/chromedp/examples
[7]: https://godoc.org/git.loafle.net/commons_go/chromedp
[7]: https://godoc.org/github.com/chromedp/chromedp
[8]: https://www.youtube.com/watch?v=_7pWCg94sKw
[9]: https://godoc.org/github.com/chromedp/cdproto
[10]: https://github.com/chromedp/cdproto-gen
[11]: https://git.loafle.net/commons_go/chromedp-proxy
[11]: https://github.com/chromedp/chromedp-proxy

23
actions.go
View File

@ -18,8 +18,8 @@ type Action interface {
type ActionFunc func(context.Context, cdp.Executor) error
// Do executes the func f using the provided context and frame handler.
func (f ActionFunc) Do(ctx context.Context, h cdp.Executor) error {
return f(ctx, h)
func (f ActionFunc) Do(ctxt context.Context, h cdp.Executor) error {
return f(ctxt, h)
}
// Tasks is a sequential list of Actions that can be used as a single Action.
@ -27,12 +27,12 @@ type Tasks []Action
// Do executes the list of Actions sequentially, using the provided context and
// frame handler.
func (t Tasks) Do(ctx context.Context, h cdp.Executor) error {
func (t Tasks) Do(ctxt context.Context, h cdp.Executor) error {
// TODO: put individual task timeouts from context here
for _, a := range t {
// ctx, cancel = context.WithTimeout(ctx, timeout)
// ctxt, cancel = context.WithTimeout(ctxt, timeout)
// defer cancel()
if err := a.Do(ctx, h); err != nil {
if err := a.Do(ctxt, h); err != nil {
return err
}
}
@ -46,15 +46,12 @@ func (t Tasks) Do(ctx context.Context, h cdp.Executor) error {
// be marked for deprecation in the future, after the remaining Actions have
// been able to be written/tested.
func Sleep(d time.Duration) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
// Don't use time.After, to avoid a temporary goroutine leak if
// ctx is cancelled before the timer fires.
t := time.NewTimer(d)
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
select {
case <-t.C:
case <-ctx.Done():
t.Stop()
return ctx.Err()
case <-time.After(d):
case <-ctxt.Done():
return ctxt.Err()
}
return nil
})

288
allocate.go
View File

@ -1,288 +0,0 @@
package chromedp
import (
"bufio"
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"strings"
"sync"
)
// An Allocator is responsible for creating and managing a number of browsers.
//
// This interface abstracts away how the browser process is actually run. For
// example, an Allocator implementation may reuse browser processes, or connect
// to already-running browsers on remote machines.
type Allocator interface {
// Allocate creates a new browser. It can be cancelled via the provided
// context, at which point all the resources used by the browser (such
// as temporary directories) will be freed.
Allocate(context.Context, ...BrowserOption) (*Browser, error)
// Wait blocks until an allocator has freed all of its resources.
// Cancelling the allocator context will already perform this operation,
// so normally there's no need to call Wait directly.
Wait()
}
// setupExecAllocator is similar to NewExecAllocator, but it allows NewContext
// to create the allocator without the unnecessary context layer.
func setupExecAllocator(opts ...ExecAllocatorOption) *ExecAllocator {
ep := &ExecAllocator{
initFlags: make(map[string]interface{}),
}
for _, o := range opts {
o(ep)
}
if ep.execPath == "" {
ep.execPath = findExecPath()
}
return ep
}
// DefaultExecAllocatorOptions are the ExecAllocator options used by NewContext
// if the given parent context doesn't have an allocator set up.
var DefaultExecAllocatorOptions = []ExecAllocatorOption{
NoFirstRun,
NoDefaultBrowserCheck,
Headless,
}
// NewExecAllocator creates a new context set up with an ExecAllocator, suitable
// for use with NewContext.
func NewExecAllocator(parent context.Context, opts ...ExecAllocatorOption) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
c := &Context{Allocator: setupExecAllocator(opts...)}
ctx = context.WithValue(ctx, contextKey{}, c)
cancelWait := func() {
cancel()
c.Allocator.Wait()
}
return ctx, cancelWait
}
// ExecAllocatorOption is a exec allocator option.
type ExecAllocatorOption func(*ExecAllocator)
// ExecAllocator is an Allocator which starts new browser processes on the host
// machine.
type ExecAllocator struct {
execPath string
initFlags map[string]interface{}
wg sync.WaitGroup
}
// Allocate satisfies the Allocator interface.
func (p *ExecAllocator) Allocate(ctx context.Context, opts ...BrowserOption) (*Browser, error) {
c := FromContext(ctx)
if c == nil {
return nil, ErrInvalidContext
}
var args []string
for name, value := range p.initFlags {
switch value := value.(type) {
case string:
args = append(args, fmt.Sprintf("--%s=%s", name, value))
case bool:
if value {
args = append(args, fmt.Sprintf("--%s", name))
}
default:
return nil, fmt.Errorf("invalid exec pool flag")
}
}
removeDir := false
dataDir, ok := p.initFlags["user-data-dir"].(string)
if !ok {
tempDir, err := ioutil.TempDir("", "chromedp-runner")
if err != nil {
return nil, err
}
args = append(args, "--user-data-dir="+tempDir)
dataDir = tempDir
removeDir = true
}
args = append(args, "--remote-debugging-port=0")
var cmd *exec.Cmd
p.wg.Add(1) // for the entire allocator
c.wg.Add(1) // for this browser's root context
go func() {
<-ctx.Done()
// First wait for the process to be finished.
if cmd != nil {
// TODO: do we care about this error in any scenario? if
// the user cancelled the context and killed chrome,
// this will most likely just be "signal: killed", which
// isn't interesting.
cmd.Wait()
}
// Then delete the temporary user data directory, if needed.
if removeDir {
if err := os.RemoveAll(dataDir); c.cancelErr == nil {
c.cancelErr = err
}
}
p.wg.Done()
c.wg.Done()
}()
// force the first page to be blank, instead of the welcome page
// TODO: why isn't --no-first-run enough?
args = append(args, "about:blank")
cmd = exec.CommandContext(ctx, p.execPath, args...)
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, err
}
// Pick up the browser's websocket URL from stderr.
wsURL := ""
scanner := bufio.NewScanner(stderr)
prefix := "DevTools listening on"
for scanner.Scan() {
line := scanner.Text()
if s := strings.TrimPrefix(line, prefix); s != line {
wsURL = strings.TrimSpace(s)
break
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
stderr.Close()
browser, err := NewBrowser(ctx, wsURL, opts...)
if err != nil {
return nil, err
}
browser.process = cmd.Process
browser.userDataDir = dataDir
return browser, nil
}
// Wait satisfies the Allocator interface.
func (p *ExecAllocator) Wait() {
p.wg.Wait()
}
// ExecPath returns an ExecAllocatorOption which uses the given path to execute
// browser processes. The given path can be an absolute path to a binary, or
// just the name of the program to find via exec.LookPath.
func ExecPath(path string) ExecAllocatorOption {
return func(p *ExecAllocator) {
if fullPath, _ := exec.LookPath(path); fullPath != "" {
// Convert to an absolute path if possible, to avoid
// repeated LookPath calls in each Allocate.
path = fullPath
}
p.execPath = path
}
}
// findExecPath tries to find the Chrome browser somewhere in the current
// system. It performs a rather agressive search, which is the same in all
// systems. That may make it a bit slow, but it will only be run when creating a
// new ExecAllocator.
func findExecPath() string {
for _, path := range [...]string{
// Unix-like
"headless_shell",
"headless-shell",
"chromium",
"chromium-browser",
"google-chrome",
"google-chrome-stable",
"google-chrome-beta",
"google-chrome-unstable",
"/usr/bin/google-chrome",
// Windows
"chrome",
"chrome.exe", // in case PATHEXT is misconfigured
`C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`,
// Mac
`/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`,
} {
found, err := exec.LookPath(path)
if err == nil {
return found
}
}
// Fall back to something simple and sensible, to give a useful error
// message.
return "google-chrome"
}
// Flag is a generic command line option to pass a flag to Chrome. If the value
// is a string, it will be passed as --name=value. If it's a boolean, it will be
// passed as --name if value is true.
func Flag(name string, value interface{}) ExecAllocatorOption {
return func(p *ExecAllocator) {
p.initFlags[name] = value
}
}
// UserDataDir is the command line option to set the user data dir.
//
// Note: set this option to manually set the profile directory used by Chrome.
// When this is not set, then a default path will be created in the /tmp
// directory.
func UserDataDir(dir string) ExecAllocatorOption {
return Flag("user-data-dir", dir)
}
// ProxyServer is the command line option to set the outbound proxy server.
func ProxyServer(proxy string) ExecAllocatorOption {
return Flag("proxy-server", proxy)
}
// WindowSize is the command line option to set the initial window size.
func WindowSize(width, height int) ExecAllocatorOption {
return Flag("window-size", fmt.Sprintf("%d,%d", width, height))
}
// UserAgent is the command line option to set the default User-Agent
// header.
func UserAgent(userAgent string) ExecAllocatorOption {
return Flag("user-agent", userAgent)
}
// NoSandbox is the Chrome comamnd line option to disable the sandbox.
func NoSandbox(p *ExecAllocator) {
Flag("no-sandbox", true)(p)
}
// NoFirstRun is the Chrome comamnd line option to disable the first run
// dialog.
func NoFirstRun(p *ExecAllocator) {
Flag("no-first-run", true)(p)
}
// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the
// default browser check.
func NoDefaultBrowserCheck(p *ExecAllocator) {
Flag("no-default-browser-check", true)(p)
}
// Headless is the command line option to run in headless mode.
func Headless(p *ExecAllocator) {
Flag("headless", true)(p)
}
// DisableGPU is the command line option to disable the GPU process.
func DisableGPU(p *ExecAllocator) {
Flag("disable-gpu", true)(p)
}

76
allocate_test.go
View File

@ -1,76 +0,0 @@
package chromedp
import (
"context"
"os"
"testing"
)
func TestExecAllocator(t *testing.T) {
t.Parallel()
allocCtx, cancel := NewExecAllocator(context.Background(), allocOpts...)
defer cancel()
// TODO: test that multiple child contexts are run in different
// processes and browsers.
taskCtx, cancel := NewContext(allocCtx)
defer cancel()
want := "insert"
var got string
if err := Run(taskCtx,
Navigate(testdataDir+"/form.html"),
Text("#foo", &got, ByID),
); err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("wanted %q, got %q", want, got)
}
cancel()
tempDir := FromContext(taskCtx).Browser.userDataDir
if _, err := os.Lstat(tempDir); !os.IsNotExist(err) {
t.Fatalf("temporary user data dir %q not deleted", tempDir)
}
}
func TestExecAllocatorCancelParent(t *testing.T) {
t.Parallel()
allocCtx, allocCancel := NewExecAllocator(context.Background(), allocOpts...)
defer allocCancel()
// TODO: test that multiple child contexts are run in different
// processes and browsers.
taskCtx, _ := NewContext(allocCtx)
if err := Run(taskCtx); err != nil {
t.Fatal(err)
}
// Canceling the pool context should stop all browsers too.
allocCancel()
tempDir := FromContext(taskCtx).Browser.userDataDir
if _, err := os.Lstat(tempDir); !os.IsNotExist(err) {
t.Fatalf("temporary user data dir %q not deleted", tempDir)
}
}
func TestSkipNewContext(t *testing.T) {
ctx, cancel := NewExecAllocator(context.Background(), allocOpts...)
defer cancel()
// Using the allocator context directly (without calling NewContext)
// should be an immediate error.
err := Run(ctx, Navigate(testdataDir+"/form.html"))
want := ErrInvalidContext
if err != want {
t.Fatalf("want error to be %q, got %q", want, err)
}
}

321
browser.go
View File

@ -1,321 +0,0 @@
package chromedp
import (
"context"
"encoding/json"
"log"
"os"
"sync/atomic"
"github.com/chromedp/cdproto"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/cdproto/target"
)
// Browser is the high-level Chrome DevTools Protocol browser manager, handling
// the browser process runner, WebSocket clients, associated targets, and
// network, page, and DOM events.
type Browser struct {
conn Transport
// next is the next message id.
next int64
// tabQueue is the queue used to create new target handlers, once a new
// tab is created and attached to. The newly created Target is sent back
// via tabResult.
tabQueue chan newTab
tabResult chan *Target
// cmdQueue is the outgoing command queue.
cmdQueue chan cmdJob
// logging funcs
logf func(string, ...interface{})
errf func(string, ...interface{})
dbgf func(string, ...interface{})
// The optional fields below are helpful for some tests.
// process can be initialized by the allocators which start a process
// when allocating a browser.
process *os.Process
// userDataDir can be initialized by the allocators which set up user
// data dirs directly.
userDataDir string
}
type newTab struct {
targetID target.ID
sessionID target.SessionID
}
type cmdJob struct {
msg *cdproto.Message
resp chan *cdproto.Message
}
// NewBrowser creates a new browser.
func NewBrowser(ctx context.Context, urlstr string, opts ...BrowserOption) (*Browser, error) {
b := &Browser{
tabQueue: make(chan newTab, 1),
tabResult: make(chan *Target, 1),
cmdQueue: make(chan cmdJob),
logf: log.Printf,
}
// apply options
for _, o := range opts {
o(b)
}
// ensure errf is set
if b.errf == nil {
b.errf = func(s string, v ...interface{}) { b.logf("ERROR: "+s, v...) }
}
// dial
var err error
b.conn, err = DialContext(ctx, ForceIP(urlstr), WithConnDebugf(b.dbgf))
if err != nil {
return nil, err
}
go b.run(ctx)
return b, nil
}
func (b *Browser) newExecutorForTarget(ctx context.Context, targetID target.ID, sessionID target.SessionID) *Target {
if targetID == "" {
panic("empty target ID")
}
if sessionID == "" {
panic("empty session ID")
}
b.tabQueue <- newTab{targetID, sessionID}
return <-b.tabResult
}
func (b *Browser) Execute(ctx context.Context, method string, params json.Marshaler, res json.Unmarshaler) error {
paramsMsg := emptyObj
if params != nil {
var err error
if paramsMsg, err = json.Marshal(params); err != nil {
return err
}
}
id := atomic.AddInt64(&b.next, 1)
ch := make(chan *cdproto.Message, 1)
b.cmdQueue <- cmdJob{
msg: &cdproto.Message{
ID: id,
Method: cdproto.MethodType(method),
Params: paramsMsg,
},
resp: ch,
}
select {
case msg := <-ch:
switch {
case msg == nil:
return ErrChannelClosed
case msg.Error != nil:
return msg.Error
case res != nil:
return json.Unmarshal(msg.Result, res)
}
case <-ctx.Done():
return ctx.Err()
}
return nil
}
type tabEvent struct {
sessionID target.SessionID
msg *cdproto.Message
}
func (b *Browser) run(ctx context.Context) {
defer b.conn.Close()
cancel := FromContext(ctx).cancel
// tabEventQueue is the queue of incoming target events, to be routed by
// their session ID.
tabEventQueue := make(chan tabEvent, 1)
// resQueue is the incoming command result queue.
resQueue := make(chan *cdproto.Message, 1)
// This goroutine continuously reads events from the websocket
// connection. The separate goroutine is needed since a websocket read
// is blocking, so it cannot be used in a select statement.
go func() {
for {
msg, err := b.conn.Read()
if err != nil {
// If the websocket failed, most likely Chrome
// was closed or crashed. Cancel the entire
// Browser context to stop all activity.
cancel()
return
}
if msg.Method == cdproto.EventRuntimeExceptionThrown {
ev := new(runtime.EventExceptionThrown)
if err := json.Unmarshal(msg.Params, ev); err != nil {
b.errf("%s", err)
continue
}
b.errf("%+v\n", ev.ExceptionDetails)
continue
}
var sessionID target.SessionID
if msg.Method == cdproto.EventTargetReceivedMessageFromTarget {
event := new(target.EventReceivedMessageFromTarget)
if err := json.Unmarshal(msg.Params, event); err != nil {
b.errf("%s", err)
continue
}
sessionID = event.SessionID
msg = new(cdproto.Message)
if err := json.Unmarshal([]byte(event.Message), msg); err != nil {
b.errf("%s", err)
continue
}
}
switch {
case msg.Method != "":
if sessionID == "" {
// TODO: are we interested in browser events?
continue
}
tabEventQueue <- tabEvent{
sessionID: sessionID,
msg: msg,
}
case msg.ID != 0:
// We can't process the response here, as it's
// another goroutine that maintans respByID.
resQueue <- msg
default:
b.errf("ignoring malformed incoming message (missing id or method): %#v", msg)
}
}
}()
// This goroutine handles tabs, as well as routing events to each tab
// via the pages map.
go func() {
// This map is only safe for use within this goroutine, so don't
// declare it as a Browser field.
pages := make(map[target.SessionID]*Target, 1024)
for {
select {
case tab := <-b.tabQueue:
if _, ok := pages[tab.sessionID]; ok {
b.errf("executor for %q already exists", tab.sessionID)
}
t := &Target{
browser: b,
TargetID: tab.targetID,
SessionID: tab.sessionID,
eventQueue: make(chan *cdproto.Message, 1024),
waitQueue: make(chan func(cur *cdp.Frame) bool, 1024),
frames: make(map[cdp.FrameID]*cdp.Frame),
logf: b.logf,
errf: b.errf,
}
go t.run(ctx)
pages[tab.sessionID] = t
b.tabResult <- t
case event := <-tabEventQueue:
page, ok := pages[event.sessionID]
if !ok {
b.errf("unknown session ID %q", event.sessionID)
continue
}
select {
case page.eventQueue <- event.msg:
default:
panic("eventQueue is full")
}
case <-ctx.Done():
return
}
}
}()
respByID := make(map[int64]chan *cdproto.Message)
// This goroutine handles sending commands to the browser, and sending
// responses back for each of these commands via respByID.
for {
select {
case res := <-resQueue:
resp, ok := respByID[res.ID]
if !ok {
b.errf("id %d not present in response map", res.ID)
continue
}
if resp != nil {
// resp could be nil, if we're not interested in
// this response; for CommandSendMessageToTarget.
resp <- res
close(resp)
}
delete(respByID, res.ID)
case q := <-b.cmdQueue:
if _, ok := respByID[q.msg.ID]; ok {
b.errf("id %d already present in response map", q.msg.ID)
continue
}
respByID[q.msg.ID] = q.resp
if q.msg.Method == "" {
// Only register the chananel in respByID;
// useful for CommandSendMessageToTarget.
continue
}
if err := b.conn.Write(q.msg); err != nil {
b.errf("%s", err)
continue
}
case <-ctx.Done():
return
}
}
}
// BrowserOption is a browser option.
type BrowserOption func(*Browser)
// WithBrowserLogf is a browser option to specify a func to receive general logging.
func WithBrowserLogf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) { b.logf = f }
}
// WithBrowserErrorf is a browser option to specify a func to receive error logging.
func WithBrowserErrorf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) { b.errf = f }
}
// WithBrowserDebugf is a browser option to specify a func to log actual
// websocket messages.
func WithBrowserDebugf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) { b.dbgf = f }
}
// WithConsolef is a browser option to specify a func to receive chrome log events.
//
// Note: NOT YET IMPLEMENTED.
func WithConsolef(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) {
}
}

661
chromedp.go
View File

@ -1,292 +1,433 @@
// Package chromedp is a high level Chrome DevTools Protocol client that
// simplifies driving browsers for scraping, unit testing, or profiling web
// pages using the CDP.
// Package chromedp is a high level Chrome Debugging Protocol domain manager
// that simplifies driving web browsers (Chrome, Safari, Edge, Android Web
// Views, and others) for scraping, unit testing, or profiling web pages.
//
// chromedp requires no third-party dependencies, implementing the async Chrome
// DevTools Protocol entirely in Go.
// chromedp requires no third-party dependencies (ie, Selenium), implementing
// the async Chrome Debugging Protocol natively.
package chromedp
import (
"context"
"errors"
"fmt"
"log"
"sync"
"time"
"github.com/chromedp/cdproto/css"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/inspector"
"github.com/chromedp/cdproto/log"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/cdproto/target"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp/client"
"github.com/chromedp/chromedp/runner"
)
// Context is attached to any context.Context which is valid for use with Run.
type Context struct {
// Allocator is used to create new browsers. It is inherited from the
// parent context when using NewContext.
Allocator Allocator
const (
// DefaultNewTargetTimeout is the default time to wait for a new target to
// be started.
DefaultNewTargetTimeout = 3 * time.Second
// Browser is the browser being used in the context. It is inherited
// from the parent context when using NewContext.
Browser *Browser
// DefaultCheckDuration is the default time to sleep between a check.
DefaultCheckDuration = 50 * time.Millisecond
// Target is the target to run actions (commands) against. It is not
// inherited from the parent context, and typically each context will
// have its own unique Target pointing to a separate browser tab (page).
Target *Target
// DefaultPoolStartPort is the default start port number.
DefaultPoolStartPort = 9000
// browserOpts holds the browser options passed to NewContext via
// WithBrowserOption, so that they can later be used when allocating a
// browser in Run.
browserOpts []BrowserOption
// DefaultPoolEndPort is the default end port number.
DefaultPoolEndPort = 10000
)
// cancel simply cancels the context that was used to start Browser.
// This is useful to stop all activity and avoid deadlocks if we detect
// that the browser was closed or happened to crash. Note that this
// cancel function doesn't do any waiting.
cancel func()
// CDP contains information for managing a Chrome process runner, low level
// JSON and websocket client, and associated network, page, and DOM handling.
type CDP struct {
// r is the chrome runner.
r *runner.Runner
// first records whether this context was the one that allocated
// Browser. This is important, because its cancellation will stop the
// entire browser handler, meaning that no further actions can be
// executed.
first bool
// opts are command line options to pass to a created runner.
opts []runner.CommandLineOption
// wg allows waiting for a target to be closed on cancellation.
wg sync.WaitGroup
// watch is the channel for new client targets.
watch <-chan client.Target
// cancelErr is the first error encountered when cancelling this
// context, for example if a browser's temporary user data directory
// couldn't be deleted.
cancelErr error
// cur is the current active target's handler.
cur cdp.Executor
// handlers is the active handlers.
handlers []*TargetHandler
// handlerMap is the map of target IDs to its active handler.
handlerMap map[string]int
// logging funcs
logf, debugf, errf func(string, ...interface{})
sync.RWMutex
}
// NewContext creates a chromedp context from the parent context. The parent
// context's Allocator is inherited, defaulting to an ExecAllocator with
// DefaultExecAllocatorOptions.
//
// If the parent context contains an allocated Browser, the child context
// inherits it, and its first Run creates a new tab on that browser. Otherwise,
// its first Run will allocate a new browser.
//
// Cancelling the returned context will close a tab or an entire browser,
// depending on the logic described above. To cancel a context while checking
// for errors, see Cancel.
func NewContext(parent context.Context, opts ...ContextOption) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
c := &Context{cancel: cancel, first: true}
if pc := FromContext(parent); pc != nil {
c.Allocator = pc.Allocator
c.Browser = pc.Browser
// don't inherit Target, so that NewContext can be used to
// create a new tab on the same browser.
c.first = c.Browser == nil
// New creates and starts a new CDP instance.
func New(ctxt context.Context, opts ...Option) (*CDP, error) {
c := &CDP{
handlers: make([]*TargetHandler, 0),
handlerMap: make(map[string]int),
logf: log.Printf,
debugf: func(string, ...interface{}) {},
errf: func(s string, v ...interface{}) { log.Printf("error: "+s, v...) },
}
// apply options
for _, o := range opts {
o(c)
}
if c.Allocator == nil {
c.Allocator = setupExecAllocator(DefaultExecAllocatorOptions...)
}
ctx = context.WithValue(ctx, contextKey{}, c)
c.wg.Add(1)
go func() {
<-ctx.Done()
if c.first {
// This is the original browser tab, so the entire
// browser will already be cleaned up elsewhere.
c.wg.Done()
return
}
if c.Target == nil {
// This is a new tab, but we didn't create it and attach
// to it yet. Nothing to do.
c.wg.Done()
return
}
// Not the original browser tab; simply detach and close it.
// We need a new context, as ctx is cancelled; use a 1s timeout.
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if id := c.Target.SessionID; id != "" {
action := target.DetachFromTarget().WithSessionID(id)
if err := action.Do(ctx, c.Browser); c.cancelErr == nil {
c.cancelErr = err
}
}
if id := c.Target.TargetID; id != "" {
action := target.CloseTarget(id)
if ok, err := action.Do(ctx, c.Browser); c.cancelErr == nil {
if !ok && err == nil {
err = fmt.Errorf("could not close target %q", id)
}
c.cancelErr = err
}
}
c.wg.Done()
}()
cancelWait := func() {
cancel()
c.wg.Wait()
}
return ctx, cancelWait
}
type contextKey struct{}
// FromContext extracts the Context data stored inside a context.Context.
func FromContext(ctx context.Context) *Context {
c, _ := ctx.Value(contextKey{}).(*Context)
return c
}
// Cancel cancels a chromedp context, waits for its resources to be cleaned up,
// and returns any error encountered during that process.
//
// Usually a "defer cancel()" will be enough for most use cases. This API is
// useful if you want to catch underlying cancel errors, such as when a
// temporary directory cannot be deleted.
func Cancel(ctx context.Context) error {
c := FromContext(ctx)
if c == nil {
return ErrInvalidContext
}
c.cancel()
c.wg.Wait()
return c.cancelErr
}
// Run runs an action against context. The provided context must be a valid
// chromedp context, typically created via NewContext.
func Run(ctx context.Context, actions ...Action) error {
c := FromContext(ctx)
// If c is nil, it's not a chromedp context.
// If c.Allocator is nil, NewContext wasn't used properly.
// If c.cancel is nil, Run is being called directly with an allocator
// context.
if c == nil || c.Allocator == nil || c.cancel == nil {
return ErrInvalidContext
}
if c.Browser == nil {
browser, err := c.Allocator.Allocate(ctx, c.browserOpts...)
if err != nil {
return err
}
c.Browser = browser
}
if c.Target == nil {
if err := c.newSession(ctx); err != nil {
return err
}
}
return Tasks(actions).Do(ctx, c.Target)
}
func (c *Context) newSession(ctx context.Context) error {
var targetID target.ID
if c.first {
// If we just allocated this browser, and it has a single page
// that's blank and not attached, use it.
infos, err := target.GetTargets().Do(ctx, c.Browser)
if err != nil {
return err
}
pages := 0
for _, info := range infos {
if info.Type == "page" && info.URL == "about:blank" && !info.Attached {
targetID = info.TargetID
pages++
}
}
if pages > 1 {
// Multiple blank pages; just in case, don't use any.
targetID = ""
if err := o(c); err != nil {
return nil, err
}
}
if targetID == "" {
// check for supplied runner, if none then create one
if c.r == nil && c.watch == nil {
var err error
targetID, err = target.CreateTarget("about:blank").Do(ctx, c.Browser)
if err != nil {
return err
}
}
sessionID, err := target.AttachToTarget(targetID).Do(ctx, c.Browser)
if err != nil {
return err
}
c.Target = c.Browser.newExecutorForTarget(ctx, targetID, sessionID)
// enable domains
for _, enable := range []Action{
log.Enable(),
runtime.Enable(),
// network.Enable(),
inspector.Enable(),
page.Enable(),
dom.Enable(),
css.Enable(),
} {
if err := enable.Do(ctx, c.Target); err != nil {
return fmt.Errorf("unable to execute %T: %v", enable, err)
}
}
return nil
}
// ContextOption is a context option.
type ContextOption func(*Context)
// WithLogf is a shortcut for WithBrowserOption(WithBrowserLogf(f)).
func WithLogf(f func(string, ...interface{})) ContextOption {
return WithBrowserOption(WithBrowserLogf(f))
}
// WithErrorf is a shortcut for WithBrowserOption(WithBrowserErrorf(f)).
func WithErrorf(f func(string, ...interface{})) ContextOption {
return WithBrowserOption(WithBrowserErrorf(f))
}
// WithDebugf is a shortcut for WithBrowserOption(WithBrowserDebugf(f)).
func WithDebugf(f func(string, ...interface{})) ContextOption {
return WithBrowserOption(WithBrowserDebugf(f))
}
// WithBrowserOption allows passing a number of browser options to the allocator
// when allocating a new browser. As such, this context option can only be used
// when NewContext is allocating a new browser.
func WithBrowserOption(opts ...BrowserOption) ContextOption {
return func(c *Context) {
if !c.first {
panic("WithBrowserOption can only be used when allocating a new browser")
}
c.browserOpts = append(c.browserOpts, opts...)
}
}
// Targets lists all the targets in the browser attached to the given context.
func Targets(ctx context.Context) ([]*target.Info, error) {
// Don't rely on Run, as that needs to be able to call Targets, and we
// don't want cyclic func calls.
c := FromContext(ctx)
if c == nil || c.Allocator == nil {
return nil, ErrInvalidContext
}
if c.Browser == nil {
browser, err := c.Allocator.Allocate(ctx, c.browserOpts...)
c.r, err = runner.Run(ctxt, c.opts...)
if err != nil {
return nil, err
}
c.Browser = browser
}
return target.GetTargets().Do(ctx, c.Browser)
// watch handlers
if c.watch == nil {
c.watch = c.r.WatchPageTargets(ctxt)
}
go func() {
for t := range c.watch {
if t == nil {
return
}
go c.AddTarget(ctxt, t)
}
}()
// TODO: fix this
timeout := time.After(defaultNewTargetTimeout)
// wait until at least one target active
for {
select {
default:
c.RLock()
exists := c.cur != nil
c.RUnlock()
if exists {
return c, nil
}
// TODO: fix this
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return nil, ctxt.Err()
case <-timeout:
return nil, errors.New("timeout waiting for initial target")
}
}
}
// AddTarget adds a target using the supplied context.
func (c *CDP) AddTarget(ctxt context.Context, t client.Target) {
c.Lock()
defer c.Unlock()
// create target manager
h, err := NewTargetHandler(t, c.logf, c.debugf, c.errf)
if err != nil {
c.errf("could not create handler for %s: %v", t, err)
return
}
// run
if err := h.Run(ctxt); err != nil {
c.errf("could not start handler for %s: %v", t, err)
return
}
// add to active handlers
c.handlers = append(c.handlers, h)
c.handlerMap[t.GetID()] = len(c.handlers) - 1
if c.cur == nil {
c.cur = h
}
}
// Wait waits for the Chrome runner to terminate.
func (c *CDP) Wait() error {
c.RLock()
r := c.r
c.RUnlock()
if r != nil {
return r.Wait()
}
return nil
}
// Shutdown closes all Chrome page handlers.
func (c *CDP) Shutdown(ctxt context.Context, opts ...client.Option) error {
c.RLock()
defer c.RUnlock()
if c.r != nil {
return c.r.Shutdown(ctxt, opts...)
}
return nil
}
// ListTargets returns the target IDs of the managed targets.
func (c *CDP) ListTargets() []string {
c.RLock()
defer c.RUnlock()
i, targets := 0, make([]string, len(c.handlers))
for k := range c.handlerMap {
targets[i] = k
i++
}
return targets
}
// GetHandlerByIndex retrieves the domains manager for the specified index.
func (c *CDP) GetHandlerByIndex(i int) cdp.Executor {
c.RLock()
defer c.RUnlock()
if i < 0 || i >= len(c.handlers) {
return nil
}
return c.handlers[i]
}
// GetHandlerByID retrieves the domains manager for the specified target ID.
func (c *CDP) GetHandlerByID(id string) cdp.Executor {
c.RLock()
defer c.RUnlock()
if i, ok := c.handlerMap[id]; ok {
return c.handlers[i]
}
return nil
}
// SetHandler sets the active handler to the target with the specified index.
func (c *CDP) SetHandler(i int) error {
c.Lock()
defer c.Unlock()
if i < 0 || i >= len(c.handlers) {
return fmt.Errorf("no handler associated with target index %d", i)
}
c.cur = c.handlers[i]
return nil
}
// SetHandlerByID sets the active target to the target with the specified id.
func (c *CDP) SetHandlerByID(id string) error {
c.Lock()
defer c.Unlock()
if i, ok := c.handlerMap[id]; ok {
c.cur = c.handlers[i]
}
return fmt.Errorf("no handler associated with target id %s", id)
}
// newTarget creates a new target using supplied context and options, returning
// the id of the created target only after the target has been started for
// monitoring.
func (c *CDP) newTarget(ctxt context.Context, opts ...client.Option) (string, error) {
c.RLock()
cl := c.r.Client(opts...)
c.RUnlock()
// new page target
t, err := cl.NewPageTarget(ctxt)
if err != nil {
return "", err
}
timeout := time.After(DefaultNewTargetTimeout)
for {
select {
default:
var ok bool
id := t.GetID()
c.RLock()
_, ok = c.handlerMap[id]
c.RUnlock()
if ok {
return id, nil
}
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return "", ctxt.Err()
case <-timeout:
return "", errors.New("timeout waiting for new target to be available")
}
}
}
// SetTarget is an action that sets the active Chrome handler to the specified
// index i.
func (c *CDP) SetTarget(i int) Action {
return ActionFunc(func(context.Context, cdp.Executor) error {
return c.SetHandler(i)
})
}
// SetTargetByID is an action that sets the active Chrome handler to the handler
// associated with the specified id.
func (c *CDP) SetTargetByID(id string) Action {
return ActionFunc(func(context.Context, cdp.Executor) error {
return c.SetHandlerByID(id)
})
}
// NewTarget is an action that creates a new Chrome target, and sets it as the
// active target.
func (c *CDP) NewTarget(id *string, opts ...client.Option) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
n, err := c.newTarget(ctxt, opts...)
if err != nil {
return err
}
if id != nil {
*id = n
}
return nil
})
}
// CloseByIndex closes the Chrome target with specified index i.
func (c *CDP) CloseByIndex(i int) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
return nil
})
}
// CloseByID closes the Chrome target with the specified id.
func (c *CDP) CloseByID(id string) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
return nil
})
}
// Run executes the action against the current target using the supplied
// context.
func (c *CDP) Run(ctxt context.Context, a Action) error {
c.RLock()
cur := c.cur
c.RUnlock()
return a.Do(ctxt, cur)
}
// Option is a Chrome Debugging Protocol option.
type Option func(*CDP) error
// WithRunner is a CDP option to specify the underlying Chrome runner to
// monitor for page handlers.
func WithRunner(r *runner.Runner) Option {
return func(c *CDP) error {
c.r = r
return nil
}
}
// WithTargets is a CDP option to specify the incoming targets to monitor for
// page handlers.
func WithTargets(watch <-chan client.Target) Option {
return func(c *CDP) error {
c.watch = watch
return nil
}
}
// WithClient is a CDP option to use the incoming targets from a client.
func WithClient(ctxt context.Context, cl *client.Client) Option {
return func(c *CDP) error {
return WithTargets(cl.WatchPageTargets(ctxt))(c)
}
}
// WithURL is a CDP option to use a client with the specified URL.
func WithURL(ctxt context.Context, urlstr string) Option {
return func(c *CDP) error {
return WithClient(ctxt, client.New(client.URL(urlstr)))(c)
}
}
// WithRunnerOptions is a CDP option to specify the options to pass to a newly
// created Chrome process runner.
func WithRunnerOptions(opts ...runner.CommandLineOption) Option {
return func(c *CDP) error {
c.opts = opts
return nil
}
}
// WithLogf is a CDP option to specify a func to receive general logging.
func WithLogf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.logf = f
return nil
}
}
// WithDebugf is a CDP option to specify a func to receive debug logging (ie,
// protocol information).
func WithDebugf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.debugf = f
return nil
}
}
// WithErrorf is a CDP option to specify a func to receive error logging.
func WithErrorf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.errf = f
return nil
}
}
// WithLog is a CDP option that sets the logging, debugging, and error funcs to
// f.
func WithLog(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.logf, c.debugf, c.errf = f, f, f
return nil
}
}
// WithConsolef is a CDP option to specify a func to receive chrome log events.
//
// Note: NOT YET IMPLEMENTED.
func WithConsolef(f func(string, ...interface{})) Option {
return func(c *CDP) error {
return nil
}
}
var (
// defaultNewTargetTimeout is the default target timeout -- used by
// testing.
defaultNewTargetTimeout = DefaultNewTargetTimeout
)

269
chromedp_test.go
View File

@ -2,226 +2,125 @@ package chromedp
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"log"
"os"
"os/exec"
"path"
"runtime"
"testing"
"time"
"github.com/chromedp/chromedp/runner"
)
var (
pool *Pool
testdataDir string
browserCtx context.Context
defaultContext, defaultCancel = context.WithCancel(context.Background())
// allocOpts is filled in TestMain
allocOpts []ExecAllocatorOption
cliOpts = []runner.CommandLineOption{
runner.NoDefaultBrowserCheck,
runner.NoFirstRun,
}
)
func testAllocate(t *testing.T, path string) (_ context.Context, cancel func()) {
// Same browser, new tab; not needing to start new chrome browsers for
// each test gives a huge speed-up.
ctx, _ := NewContext(browserCtx)
func testAllocate(t *testing.T, path string) *Res {
c, err := pool.Allocate(defaultContext, cliOpts...)
if err != nil {
t.Fatalf("could not allocate from pool: %v", err)
}
err = WithLogf(t.Logf)(c.c)
if err != nil {
t.Fatalf("could not set logf: %v", err)
}
err = WithDebugf(t.Logf)(c.c)
if err != nil {
t.Fatalf("could not set debugf: %v", err)
}
err = WithErrorf(t.Errorf)(c.c)
if err != nil {
t.Fatalf("could not set errorf: %v", err)
}
h := c.c.GetHandlerByIndex(0)
th, ok := h.(*TargetHandler)
if !ok {
t.Fatalf("handler is invalid type")
}
th.logf, th.debugf = t.Logf, t.Logf
th.errf = func(s string, v ...interface{}) {
t.Logf("TARGET HANDLER ERROR: "+s, v...)
}
// Only navigate if we want a path, otherwise leave the blank page.
if path != "" {
if err := Run(ctx, Navigate(testdataDir+"/"+path)); err != nil {
t.Fatal(err)
err = c.Run(defaultContext, Navigate(testdataDir+"/"+path))
if err != nil {
t.Fatalf("could not navigate to testdata/%s: %v", path, err)
}
}
cancelErr := func() {
if err := Cancel(ctx); err != nil {
t.Error(err)
}
}
return ctx, cancelErr
return c
}
func TestMain(m *testing.M) {
var err error
wd, err := os.Getwd()
if err != nil {
panic(fmt.Sprintf("could not get working directory: %v", err))
log.Fatalf("could not get working directory: %v", err)
os.Exit(1)
}
testdataDir = "file://" + path.Join(wd, "testdata")
// build on top of the default options
allocOpts = append(allocOpts, DefaultExecAllocatorOptions...)
// disabling the GPU helps portability with some systems like Travis,
// and can slightly speed up the tests on other systems
allocOpts = append(allocOpts, DisableGPU)
// it's worth noting that newer versions of chrome (64+) run much faster
// its worth noting that newer versions of chrome (64+) run much faster
// than older ones -- same for headless_shell ...
if execPath := os.Getenv("CHROMEDP_TEST_RUNNER"); execPath != "" {
allocOpts = append(allocOpts, ExecPath(execPath))
execPath := runner.DefaultChromePath
if testRunner := os.Getenv("CHROMEDP_TEST_RUNNER"); testRunner != "" {
execPath = testRunner
} else {
// use headless_shell, if on path
var hsPath string
hsPath, err = exec.LookPath("headless_shell")
if err == nil {
execPath = hsPath
}
}
cliOpts = append(cliOpts, runner.ExecPath(execPath))
// not explicitly needed to be set, as this vastly speeds up unit tests
if noSandbox := os.Getenv("CHROMEDP_NO_SANDBOX"); noSandbox != "false" {
allocOpts = append(allocOpts, NoSandbox)
cliOpts = append(cliOpts, runner.NoSandbox)
}
// must be explicitly set, as disabling gpu slows unit tests
if disableGPU := os.Getenv("CHROMEDP_DISABLE_GPU"); disableGPU != "" && disableGPU != "false" {
cliOpts = append(cliOpts, runner.DisableGPU)
}
allocCtx, cancel := NewExecAllocator(context.Background(), allocOpts...)
if targetTimeout := os.Getenv("CHROMEDP_TARGET_TIMEOUT"); targetTimeout != "" {
defaultNewTargetTimeout, _ = time.ParseDuration(targetTimeout)
}
if defaultNewTargetTimeout == 0 {
defaultNewTargetTimeout = 30 * time.Second
}
// start the browser
browserCtx, _ = NewContext(allocCtx)
if err := Run(browserCtx); err != nil {
panic(err)
//pool, err = NewPool(PoolLog(log.Printf, log.Printf, log.Printf))
pool, err = NewPool()
if err != nil {
log.Fatal(err)
}
code := m.Run()
cancel()
defaultCancel()
err = pool.Shutdown()
if err != nil {
log.Fatal(err)
}
os.Exit(code)
}
func TestTargets(t *testing.T) {
t.Parallel()
// Start one browser with one tab.
ctx1, cancel1 := NewContext(context.Background())
defer cancel1()
if err := Run(ctx1); err != nil {
t.Fatal(err)
}
wantTargets := func(ctx context.Context, want int) {
t.Helper()
infos, err := Targets(ctx)
if err != nil {
t.Fatal(err)
}
if got := len(infos); want != got {
t.Fatalf("want %d targets, got %d", want, got)
}
}
wantTargets(ctx1, 1)
// Start a second tab on the same browser.
ctx2, cancel2 := NewContext(ctx1)
defer cancel2()
if err := Run(ctx2); err != nil {
t.Fatal(err)
}
wantTargets(ctx2, 2)
// The first context should also see both targets.
wantTargets(ctx1, 2)
// Cancelling the second context should close the second tab alone.
cancel2()
wantTargets(ctx1, 1)
// We used to have a bug where Run would reset the first context as if
// it weren't the first, breaking its cancellation.
if err := Run(ctx1); err != nil {
t.Fatal(err)
}
}
func TestBrowserQuit(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("os.Interrupt isn't supported on Windows")
}
// Simulate a scenario where we navigate to a page that's slow to
// respond, and the browser is closed before we can finish the
// navigation.
serve := make(chan bool, 1)
close := make(chan bool, 1)
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close <- true
<-serve
fmt.Fprintf(w, "response")
}))
defer s.Close()
ctx, cancel := NewContext(context.Background())
defer cancel()
if err := Run(ctx); err != nil {
t.Fatal(err)
}
go func() {
<-close
b := FromContext(ctx).Browser
if err := b.process.Signal(os.Interrupt); err != nil {
t.Error(err)
}
serve <- true
}()
// Run should error with something other than "deadline exceeded" in
// much less than 5s.
ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
switch err := Run(ctx2, Navigate(s.URL)); err {
case nil:
t.Fatal("did not expect a nil error")
case context.DeadlineExceeded:
t.Fatalf("did not expect a standard context error: %v", err)
}
}
func TestCancelError(t *testing.T) {
t.Parallel()
ctx1, cancel1 := NewContext(context.Background())
defer cancel1()
if err := Run(ctx1); err != nil {
t.Fatal(err)
}
// Open and close a target normally; no error.
ctx2, cancel2 := NewContext(ctx1)
defer cancel2()
if err := Run(ctx2); err != nil {
t.Fatal(err)
}
if err := Cancel(ctx2); err != nil {
t.Fatalf("expected a nil error, got %v", err)
}
// Make "cancel" close the wrong target; error.
ctx3, cancel3 := NewContext(ctx1)
defer cancel3()
if err := Run(ctx3); err != nil {
t.Fatal(err)
}
FromContext(ctx3).Target.TargetID = "wrong"
if err := Cancel(ctx3); err == nil {
t.Fatalf("expected a non-nil error, got %v", err)
}
}
func TestPrematureCancel(t *testing.T) {
t.Parallel()
// Cancel before the browser is allocated.
ctx, cancel := NewContext(context.Background())
cancel()
if err := Run(ctx); err != context.Canceled {
t.Fatalf("wanted canceled context error, got %v", err)
}
}
func TestPrematureCancelTab(t *testing.T) {
t.Parallel()
ctx1, cancel := NewContext(context.Background())
defer cancel()
if err := Run(ctx1); err != nil {
t.Fatal(err)
}
// Cancel after the browser is allocated, but before we've created a new
// tab.
ctx2, cancel := NewContext(ctx1)
cancel()
Run(ctx2)
}

40
client/chrome.go Normal file
View File

@ -0,0 +1,40 @@
package client
import "fmt"
//go:generate easyjson -omit_empty -output_filename easyjson.go chrome.go
// Chrome holds connection information for a Chrome, Edge, or Safari target.
//
//easyjson:json
type Chrome struct {
Description string `json:"description,omitempty"`
DevtoolsURL string `json:"devtoolsFrontendUrl,omitempty"`
ID string `json:"id,omitempty"`
Title string `json:"title,omitempty"`
Type TargetType `json:"type,omitempty"`
URL string `json:"url,omitempty"`
WebsocketURL string `json:"webSocketDebuggerUrl,omitempty"`
FaviconURL string `json:"faviconURL,omitempty"`
}
// String satisfies the stringer interface.
func (c Chrome) String() string {
return fmt.Sprintf("%s (`%s`)", c.ID, c.Title)
}
// GetID returns the target ID.
func (c *Chrome) GetID() string {
return c.ID
}
// GetType returns the target type.
func (c *Chrome) GetType() TargetType {
return c.Type
}
// GetWebsocketURL provides the websocket URL for the target, satisfying the
// domains.Target interface.
func (c *Chrome) GetWebsocketURL() string {
return c.WebsocketURL
}

350
client/client.go Normal file
View File

@ -0,0 +1,350 @@
// Package client provides the low level Chrome Debugging Protocol JSON types
// and related funcs.
package client
//go:generate go run gen.go
import (
"context"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"
"sync"
"time"
"github.com/mailru/easyjson"
)
const (
// DefaultEndpoint is the default endpoint to connect to.
DefaultEndpoint = "http://localhost:9222/json"
// DefaultWatchInterval is the default check duration.
DefaultWatchInterval = 100 * time.Millisecond
// DefaultWatchTimeout is the default watch timeout.
DefaultWatchTimeout = 5 * time.Second
)
// Error is a client error.
type Error string
// Error satisfies the error interface.
func (err Error) Error() string {
return string(err)
}
const (
// ErrUnsupportedProtocolType is the unsupported protocol type error.
ErrUnsupportedProtocolType Error = "unsupported protocol type"
// ErrUnsupportedProtocolVersion is the unsupported protocol version error.
ErrUnsupportedProtocolVersion Error = "unsupported protocol version"
)
// Target is the common interface for a Chrome Debugging Protocol target.
type Target interface {
String() string
GetID() string
GetType() TargetType
GetWebsocketURL() string
}
// Client is a Chrome Debugging Protocol client.
type Client struct {
url string
check time.Duration
timeout time.Duration
ver, typ string
rw sync.RWMutex
}
// New creates a new Chrome Debugging Protocol client.
func New(opts ...Option) *Client {
c := &Client{
url: DefaultEndpoint,
check: DefaultWatchInterval,
timeout: DefaultWatchTimeout,
}
// apply opts
for _, o := range opts {
o(c)
}
return c
}
// doReq executes a request.
func (c *Client) doReq(ctxt context.Context, action string, v interface{}) error {
// create request
req, err := http.NewRequest("GET", c.url+"/"+action, nil)
if err != nil {
return err
}
req = req.WithContext(ctxt)
cl := &http.Client{}
// execute
res, err := cl.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if v != nil {
// load body
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
// unmarshal
if z, ok := v.(easyjson.Unmarshaler); ok {
return easyjson.Unmarshal(body, z)
}
return json.Unmarshal(body, v)
}
return nil
}
// ListTargets returns a list of all targets.
func (c *Client) ListTargets(ctxt context.Context) ([]Target, error) {
var err error
var l []json.RawMessage
err = c.doReq(ctxt, "list", &l)
if err != nil {
return nil, err
}
t := make([]Target, len(l))
for i, v := range l {
t[i], err = c.newTarget(ctxt, v)
if err != nil {
return nil, err
}
}
return t, nil
}
// ListTargetsWithType returns a list of Targets with the specified target
// type.
func (c *Client) ListTargetsWithType(ctxt context.Context, typ TargetType) ([]Target, error) {
var err error
targets, err := c.ListTargets(ctxt)
if err != nil {
return nil, err
}
var ret []Target
for _, t := range targets {
if t.GetType() == typ {
ret = append(ret, t)
}
}
return ret, nil
}
// ListPageTargets lists the available Page targets.
func (c *Client) ListPageTargets(ctxt context.Context) ([]Target, error) {
return c.ListTargetsWithType(ctxt, Page)
}
var browserRE = regexp.MustCompile(`(?i)^(chrome|chromium|microsoft edge|safari)`)
// loadProtocolInfo loads the protocol information from the remote URL.
func (c *Client) loadProtocolInfo(ctxt context.Context) (string, string, error) {
c.rw.Lock()
defer c.rw.Unlock()
if c.ver == "" || c.typ == "" {
v, err := c.VersionInfo(ctxt)
if err != nil {
return "", "", err
}
if m := browserRE.FindAllStringSubmatch(v["Browser"], -1); len(m) != 0 {
c.typ = strings.ToLower(m[0][0])
}
c.ver = v["Protocol-Version"]
}
return c.ver, c.typ, nil
}
// newTarget creates a new target.
func (c *Client) newTarget(ctxt context.Context, buf []byte) (Target, error) {
var err error
ver, typ, err := c.loadProtocolInfo(ctxt)
if err != nil {
return nil, err
}
if ver != "1.1" && ver != "1.2" && ver != "1.3" {
return nil, ErrUnsupportedProtocolVersion
}
switch typ {
case "chrome", "chromium", "microsoft edge", "safari", "":
x := new(Chrome)
if buf != nil {
err = easyjson.Unmarshal(buf, x)
if err != nil {
return nil, err
}
}
return x, nil
}
return nil, ErrUnsupportedProtocolType
}
// NewPageTargetWithURL creates a new page target with the specified url.
func (c *Client) NewPageTargetWithURL(ctxt context.Context, urlstr string) (Target, error) {
var err error
t, err := c.newTarget(ctxt, nil)
if err != nil {
return nil, err
}
u := "new"
if urlstr != "" {
u += "?" + urlstr
}
err = c.doReq(ctxt, u, t)
if err != nil {
return nil, err
}
return t, nil
}
// NewPageTarget creates a new page target.
func (c *Client) NewPageTarget(ctxt context.Context) (Target, error) {
return c.NewPageTargetWithURL(ctxt, "")
}
// ActivateTarget activates a target.
func (c *Client) ActivateTarget(ctxt context.Context, t Target) error {
return c.doReq(ctxt, "activate/"+t.GetID(), nil)
}
// CloseTarget activates a target.
func (c *Client) CloseTarget(ctxt context.Context, t Target) error {
return c.doReq(ctxt, "close/"+t.GetID(), nil)
}
// VersionInfo returns information about the remote debugging protocol.
func (c *Client) VersionInfo(ctxt context.Context) (map[string]string, error) {
var err error
v := map[string]string{}
err = c.doReq(ctxt, "version", &v)
if err != nil {
return nil, err
}
return v, nil
}
// WatchPageTargets watches for new page targets.
func (c *Client) WatchPageTargets(ctxt context.Context) <-chan Target {
if ctxt == nil {
ctxt = context.Background()
}
ch := make(chan Target)
go func() {
defer close(ch)
encountered := make(map[string]bool)
check := func() error {
targets, err := c.ListPageTargets(ctxt)
if err != nil {
return err
}
for _, t := range targets {
if !encountered[t.GetID()] {
ch <- t
}
encountered[t.GetID()] = true
}
return nil
}
var err error
lastGood := time.Now()
for {
err = check()
if err == nil {
lastGood = time.Now()
} else if time.Now().After(lastGood.Add(c.timeout)) {
return
}
select {
case <-time.After(c.check):
continue
case <-ctxt.Done():
return
}
}
}()
return ch
}
// Option is a Chrome Debugging Protocol client option.
type Option func(*Client)
// URL is a client option to specify the remote Chrome instance to connect to.
func URL(urlstr string) Option {
return func(c *Client) {
// since chrome 66+, dev tools requires the host name to be either an
// IP address, or "localhost"
if strings.HasPrefix(strings.ToLower(urlstr), "http://") {
host, port, path := urlstr[7:], "", ""
if i := strings.Index(host, "/"); i != -1 {
host, path = host[:i], host[i:]
}
if i := strings.Index(host, ":"); i != -1 {
host, port = host[:i], host[i:]
}
if addr, err := net.ResolveIPAddr("ip", host); err == nil {
urlstr = "http://" + addr.IP.String() + port + path
}
}
c.url = urlstr
}
}
// WatchInterval is a client option that specifies the check interval duration.
func WatchInterval(check time.Duration) Option {
return func(c *Client) {
c.check = check
}
}
// WatchTimeout is a client option that specifies the watch timeout duration.
func WatchTimeout(timeout time.Duration) Option {
return func(c *Client) {
c.timeout = timeout
}
}

174
client/easyjson.go Normal file
View File

@ -0,0 +1,174 @@
// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT.
package client
import (
json "encoding/json"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// suppress unused package warning
var (
_ *json.RawMessage
_ *jlexer.Lexer
_ *jwriter.Writer
_ easyjson.Marshaler
)
func easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(in *jlexer.Lexer, out *Chrome) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "description":
out.Description = string(in.String())
case "devtoolsFrontendUrl":
out.DevtoolsURL = string(in.String())
case "id":
out.ID = string(in.String())
case "title":
out.Title = string(in.String())
case "type":
(out.Type).UnmarshalEasyJSON(in)
case "url":
out.URL = string(in.String())
case "webSocketDebuggerUrl":
out.WebsocketURL = string(in.String())
case "faviconURL":
out.FaviconURL = string(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(out *jwriter.Writer, in Chrome) {
out.RawByte('{')
first := true
_ = first
if in.Description != "" {
const prefix string = ",\"description\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Description))
}
if in.DevtoolsURL != "" {
const prefix string = ",\"devtoolsFrontendUrl\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.DevtoolsURL))
}
if in.ID != "" {
const prefix string = ",\"id\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.ID))
}
if in.Title != "" {
const prefix string = ",\"title\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Title))
}
if in.Type != "" {
const prefix string = ",\"type\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
(in.Type).MarshalEasyJSON(out)
}
if in.URL != "" {
const prefix string = ",\"url\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.URL))
}
if in.WebsocketURL != "" {
const prefix string = ",\"webSocketDebuggerUrl\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.WebsocketURL))
}
if in.FaviconURL != "" {
const prefix string = ",\"faviconURL\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.FaviconURL))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v Chrome) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v Chrome) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *Chrome) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *Chrome) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(l, v)
}

142
client/gen.go Normal file
View File

@ -0,0 +1,142 @@
// +build ignore
package main
import (
"encoding/base64"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os/exec"
"regexp"
"sort"
"github.com/knq/snaker"
)
const (
// chromiumSrc is the base chromium source repo location
chromiumSrc = "https://chromium.googlesource.com/chromium/src"
// devtoolsHTTPClientCc contains the target_type names.
devtoolsHTTPClientCc = chromiumSrc + "/+/master/chrome/test/chromedriver/chrome/devtools_http_client.cc?format=TEXT"
)
var (
flagOut = flag.String("out", "targettype.go", "out file")
typeAsStringRE = regexp.MustCompile(`type_as_string\s+==\s+"([^"]+)"`)
)
func main() {
flag.Parse()
// grab source
buf, err := grab(devtoolsHTTPClientCc)
if err != nil {
log.Fatal(err)
}
// find names
matches := typeAsStringRE.FindAllStringSubmatch(string(buf), -1)
names := make([]string, len(matches))
for i, m := range matches {
names[i] = m[1]
}
sort.Strings(names)
// process names
var constVals, decodeVals string
for _, n := range names {
name := snaker.SnakeToCamelIdentifier(n)
constVals += fmt.Sprintf("%s TargetType = \"%s\"\n", name, n)
decodeVals += fmt.Sprintf("case %s:\n*tt=%s\n", name, name)
}
err = ioutil.WriteFile(*flagOut, []byte(fmt.Sprintf(targetTypeSrc, constVals, decodeVals)), 0644)
if err != nil {
log.Fatal(err)
}
err = exec.Command("gofmt", "-w", "-s", *flagOut).Run()
if err != nil {
log.Fatal(err)
}
}
// grab retrieves a file from the chromium source code.
func grab(path string) ([]byte, error) {
res, err := http.Get(path)
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
buf, err := base64.StdEncoding.DecodeString(string(body))
if err != nil {
return nil, err
}
return buf, nil
}
const (
targetTypeSrc = `package client
// Code generated by gen.go. DO NOT EDIT.
import (
// "errors"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// TargetType are the types of targets available in Chrome.
type TargetType string
// TargetType values.
const (
%s
)
// String satisfies stringer.
func (tt TargetType) String() string {
return string(tt)
}
// MarshalEasyJSON satisfies easyjson.Marshaler.
func (tt TargetType) MarshalEasyJSON(out *jwriter.Writer) {
out.String(string(tt))
}
// MarshalJSON satisfies json.Marshaler.
func (tt TargetType) MarshalJSON() ([]byte, error) {
return easyjson.Marshal(tt)
}
// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
z := TargetType(in.String())
switch z {
%s
default:
// in.AddError(errors.New("unknown TargetType"))
*tt = z
}
}
// UnmarshalJSON satisfies json.Unmarshaler.
func (tt *TargetType) UnmarshalJSON(buf []byte) error {
return easyjson.Unmarshal(buf, tt)
}
`
)

81
client/targettype.go Normal file
View File

@ -0,0 +1,81 @@
package client
// Code generated by gen.go. DO NOT EDIT.
import (
// "errors"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// TargetType are the types of targets available in Chrome.
type TargetType string
// TargetType values.
const (
App TargetType = "app"
BackgroundPage TargetType = "background_page"
Browser TargetType = "browser"
External TargetType = "external"
Iframe TargetType = "iframe"
Other TargetType = "other"
Page TargetType = "page"
ServiceWorker TargetType = "service_worker"
SharedWorker TargetType = "shared_worker"
Webview TargetType = "webview"
Worker TargetType = "worker"
)
// String satisfies stringer.
func (tt TargetType) String() string {
return string(tt)
}
// MarshalEasyJSON satisfies easyjson.Marshaler.
func (tt TargetType) MarshalEasyJSON(out *jwriter.Writer) {
out.String(string(tt))
}
// MarshalJSON satisfies json.Marshaler.
func (tt TargetType) MarshalJSON() ([]byte, error) {
return easyjson.Marshal(tt)
}
// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
z := TargetType(in.String())
switch z {
case App:
*tt = App
case BackgroundPage:
*tt = BackgroundPage
case Browser:
*tt = Browser
case External:
*tt = External
case Iframe:
*tt = Iframe
case Other:
*tt = Other
case Page:
*tt = Page
case ServiceWorker:
*tt = ServiceWorker
case SharedWorker:
*tt = SharedWorker
case Webview:
*tt = Webview
case Worker:
*tt = Worker
default:
// in.AddError(errors.New("unknown TargetType"))
*tt = z
}
}
// UnmarshalJSON satisfies json.Unmarshaler.
func (tt *TargetType) UnmarshalJSON(buf []byte) error {
return easyjson.Unmarshal(buf, tt)
}

69
client/transport.go Normal file
View File

@ -0,0 +1,69 @@
package client
import (
"io"
"github.com/gorilla/websocket"
)
const (
// DefaultReadBufferSize is the default maximum read buffer size.
DefaultReadBufferSize = 25 * 1024 * 1024
// DefaultWriteBufferSize is the default maximum write buffer size.
DefaultWriteBufferSize = 10 * 1024 * 1024
)
// Transport is the common interface to send/receive messages.
type Transport interface {
Read() ([]byte, error)
Write([]byte) error
io.Closer
}
// Conn wraps a gorilla/websocket.Conn connection.
type Conn struct {
*websocket.Conn
}
// Read reads the next websocket message.
func (c *Conn) Read() ([]byte, error) {
_, buf, err := c.ReadMessage()
if err != nil {
return nil, err
}
return buf, nil
}
// Write writes a websocket message.
func (c *Conn) Write(buf []byte) error {
return c.WriteMessage(websocket.TextMessage, buf)
}
// Dial dials the specified target's websocket URL.
//
// Note: uses gorilla/websocket.
func Dial(t Target, opts ...DialOption) (Transport, error) {
d := &websocket.Dialer{
ReadBufferSize: DefaultReadBufferSize,
WriteBufferSize: DefaultWriteBufferSize,
}
// apply opts
for _, o := range opts {
o(d)
}
// connect
conn, _, err := d.Dial(t.GetWebsocketURL(), nil)
if err != nil {
return nil, err
}
return &Conn{conn}, nil
}
// DialOption is a dial option.
type DialOption func(*websocket.Dialer)
// TODO: add dial options ...

152
conn.go
View File

@ -1,152 +0,0 @@
package chromedp
import (
"context"
"io"
"io/ioutil"
"net"
"strings"
"github.com/chromedp/cdproto"
"github.com/gorilla/websocket"
"github.com/mailru/easyjson"
)
var (
// DefaultReadBufferSize is the default maximum read buffer size.
DefaultReadBufferSize = 25 * 1024 * 1024
// DefaultWriteBufferSize is the default maximum write buffer size.
DefaultWriteBufferSize = 10 * 1024 * 1024
)
// Transport is the common interface to send/receive messages to a target.
type Transport interface {
Read() (*cdproto.Message, error)
Write(*cdproto.Message) error
io.Closer
}
// Conn wraps a gorilla/websocket.Conn connection.
type Conn struct {
*websocket.Conn
dbgf func(string, ...interface{})
}
// DialContext dials the specified websocket URL using gorilla/websocket.
func DialContext(ctx context.Context, urlstr string, opts ...DialOption) (*Conn, error) {
d := &websocket.Dialer{
ReadBufferSize: DefaultReadBufferSize,
WriteBufferSize: DefaultWriteBufferSize,
}
// connect
conn, _, err := d.DialContext(ctx, urlstr, nil)
if err != nil {
return nil, err
}
// apply opts
c := &Conn{
Conn: conn,
}
for _, o := range opts {
o(c)
}
return c, nil
}
// Read reads the next message.
func (c *Conn) Read() (*cdproto.Message, error) {
// get websocket reader
typ, r, err := c.NextReader()
if err != nil {
return nil, err
}
if typ != websocket.TextMessage {
return nil, ErrInvalidWebsocketMessage
}
// when dbgf defined, buffer, log, unmarshal
if c.dbgf != nil {
// buffer output
buf, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
c.dbgf("<- %s", string(buf))
msg := new(cdproto.Message)
if err = easyjson.Unmarshal(buf, msg); err != nil {
return nil, err
}
return msg, nil
}
// unmarshal direct from reader
msg := new(cdproto.Message)
if err = easyjson.UnmarshalFromReader(r, msg); err != nil {
return nil, err
}
return msg, nil
}
// Write writes a message.
func (c *Conn) Write(msg *cdproto.Message) error {
w, err := c.NextWriter(websocket.TextMessage)
if err != nil {
return err
}
if c.dbgf != nil {
var buf []byte
buf, err = easyjson.Marshal(msg)
if err != nil {
return err
}
c.dbgf("-> %s", string(buf))
_, err = w.Write(buf)
if err != nil {
return err
}
} else {
// direct marshal
_, err = easyjson.MarshalToWriter(msg, w)
if err != nil {
return err
}
}
return w.Close()
}
// ForceIP forces the host component in urlstr to be an IP address.
//
// Since Chrome 66+, Chrome DevTools Protocol clients connecting to a browser
// must send the "Host:" header as either an IP address, or "localhost".
func ForceIP(urlstr string) string {
if i := strings.Index(urlstr, "://"); i != -1 {
scheme := urlstr[:i+3]
host, port, path := urlstr[len(scheme)+3:], "", ""
if i := strings.Index(host, "/"); i != -1 {
host, path = host[:i], host[i:]
}
if i := strings.Index(host, ":"); i != -1 {
host, port = host[:i], host[i:]
}
if addr, err := net.ResolveIPAddr("ip", host); err == nil {
urlstr = scheme + addr.IP.String() + port + path
}
}
return urlstr
}
// DialOption is a dial option.
type DialOption func(*Conn)
// WithConnDebugf is a dial option to set a protocol logger.
func WithConnDebugf(f func(string, ...interface{})) DialOption {
return func(c *Conn) {
c.dbgf = f
}
}

34
contrib/meta.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash
SRC=$(realpath $(cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../)
pushd $SRC &> /dev/null
gometalinter \
--disable=aligncheck \
--enable=misspell \
--enable=gofmt \
--deadline=100s \
--cyclo-over=25 \
--sort=path \
--exclude='\(defer (.+?)\)\) \(errcheck\)$' \
--exclude='/easyjson\.go.*(passes|copies) lock' \
--exclude='/easyjson\.go.*ineffectual assignment' \
--exclude='/easyjson\.go.*unnecessary conversion' \
--exclude='/easyjson\.go.*this value of key is never used' \
--exclude='/easyjson\.go.*\((gocyclo|golint|goconst|staticcheck)\)$' \
--exclude='^cdp/.*Potential hardcoded credentials' \
--exclude='^cdp/cdp\.go.*UnmarshalEasyJSON.*\(gocyclo\)$' \
--exclude='^cdp/cdputil/cdputil\.go.*UnmarshalMessage.*\(gocyclo\)$' \
--exclude='^cmd/chromedp-gen/.*\((gocyclo|interfacer)\)$' \
--exclude='^cmd/chromedp-proxy/main\.go.*\(gas\)$' \
--exclude='^cmd/chromedp-gen/fixup/fixup\.go.*\(goconst\)$' \
--exclude='^cmd/chromedp-gen/internal/enum\.go.*unreachable' \
--exclude='^cmd/chromedp-gen/(main|domain-gen)\.go.*\(gas\)$' \
--exclude='^examples/[a-z]+/main\.go.*\(errcheck\)$' \
--exclude='^kb/gen\.go.*\((gas|vet)\)$' \
--exclude='^runner/.*\(gas\)$' \
--exclude='^handler\.go.*cmd can be easyjson\.Marshaler' \
./...
popd &> /dev/null

10
contrib/start.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
TMP=$(mktemp -d /tmp/google-chrome.XXXXX)
google-chrome \
--user-data-dir=$TMP \
--remote-debugging-port=9222 \
--no-first-run \
--no-default-browser-check \
about:blank

14
contrib/stats.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
BASE=$(realpath $(cd -P $GOPATH/src/github.com/chromedp && pwd))
FILES=$(find $BASE/{chromedp*,goquery,examples} -type f -iname \*.go -not -iname \*.qtpl.go -print0|wc -l --files0-from=-|head -n -1)$'\n'
AUTOG=$(find $BASE/cdproto/ -type f -iname \*.go -not -iname \*easyjson\* -print0|wc -l --files0-from=-|head -n -1)
if [ "$1" != "--total" ]; then
echo -e "code:\n$FILES\n\ngenerated:\n$AUTOG"
else
echo "code: $(awk '{s+=$1} END {print s}' <<< "$FILES")"
echo "generated: $(awk '{s+=$1} END {print s}' <<< "$AUTOG")"
fi

6
errors.go
View File

@ -10,9 +10,6 @@ func (err Error) Error() string {
// Error types.
const (
// ErrInvalidWebsocketMessage is the invalid websocket message.
ErrInvalidWebsocketMessage Error = "invalid websocket message"
// ErrInvalidDimensions is the invalid dimensions error.
ErrInvalidDimensions Error = "invalid dimensions"
@ -42,7 +39,4 @@ const (
// ErrInvalidHandler is the invalid handler error.
ErrInvalidHandler Error = "invalid handler"
// ErrInvalidContext is the invalid context error.
ErrInvalidContext Error = "invalid context"
)

6
eval.go
View File

@ -11,7 +11,7 @@ import (
// Evaluate is an action to evaluate the Javascript expression, unmarshaling
// the result of the script evaluation to res.
//
// When res is a type other than *[]byte, or **chromedp/cdproto/runtime.RemoteObject,
// When res is a type other than *[]byte, or **chromedp/cdp/runtime.RemoteObject,
// then the result of the script evaluation will be returned "by value" (ie,
// JSON-encoded), and subsequently an attempt will be made to json.Unmarshal
// the script result to res.
@ -27,7 +27,7 @@ func Evaluate(expression string, res interface{}, opts ...EvaluateOption) Action
panic("res cannot be nil")
}
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
// set up parameters
p := runtime.Evaluate(expression)
switch res.(type) {
@ -42,7 +42,7 @@ func Evaluate(expression string, res interface{}, opts ...EvaluateOption) Action
}
// evaluate
v, exp, err := p.Do(ctx, h)
v, exp, err := p.Do(ctxt, h)
if err != nil {
return err
}

100
example_test.go
View File

@ -1,100 +0,0 @@
package chromedp_test
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"git.loafle.net/commons_go/chromedp"
)
func ExampleTitle() {
ctx, cancel := chromedp.NewContext(context.Background())
defer cancel()
var title string
if err := chromedp.Run(ctx,
chromedp.Navigate("https://git.loafle.net/commons_go/chromedp/issues"),
chromedp.WaitVisible("#start-of-content", chromedp.ByID),
chromedp.Title(&title),
); err != nil {
panic(err)
}
fmt.Println(title)
// no expected output, to not run this test as part of 'go test'; it's
// too slow, requiring internet access.
}
func ExampleExecAllocator() {
dir, err := ioutil.TempDir("", "chromedp-example")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
opts := []chromedp.ExecAllocatorOption{
chromedp.NoFirstRun,
chromedp.NoDefaultBrowserCheck,
chromedp.Headless,
chromedp.DisableGPU,
chromedp.UserDataDir(dir),
}
allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
// also set up a custom logger
taskCtx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
defer cancel()
// ensure that the browser process is started
if err := chromedp.Run(taskCtx); err != nil {
panic(err)
}
path := filepath.Join(dir, "DevToolsActivePort")
bs, err := ioutil.ReadFile(path)
if err != nil {
panic(err)
}
lines := bytes.Split(bs, []byte("\n"))
fmt.Printf("DevToolsActivePort has %d lines\n", len(lines))
// Output:
// DevToolsActivePort has 2 lines
}
func ExampleNewContext_manyTabs() {
// new browser, first tab
ctx1, cancel := chromedp.NewContext(context.Background())
defer cancel()
// ensure the first tab is created
if err := chromedp.Run(ctx1); err != nil {
panic(err)
}
// same browser, second tab
ctx2, _ := chromedp.NewContext(ctx1)
// ensure the second tab is created
if err := chromedp.Run(ctx2); err != nil {
panic(err)
}
c1 := chromedp.FromContext(ctx1)
c2 := chromedp.FromContext(ctx2)
fmt.Printf("Same browser: %t\n", c1.Browser == c2.Browser)
fmt.Printf("Same tab: %t\n", c1.Target == c2.Target)
// Output:
// Same browser: true
// Same tab: false
}

14
go.mod
View File

@ -1,10 +1,10 @@
module git.loafle.net/commons_go/chromedp
go 1.12
module github.com/chromedp/chromedp
require (
github.com/chromedp/cdproto v0.0.0-20190412020601-c4267f5c421a
github.com/disintegration/imaging v1.6.0
github.com/gorilla/websocket v1.4.0
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983
github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3
github.com/disintegration/imaging v1.4.2
github.com/gorilla/websocket v1.2.0
github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794
github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856
golang.org/x/image v0.0.0-20180628062038-cc896f830ced
)

22
go.sum
View File

@ -1,13 +1,9 @@
github.com/chromedp/cdproto v0.0.0-20190412020601-c4267f5c421a h1:GZPhzysmNSpFnYVSzixFV/ECNILkkn5HJon7AOUNizg=
github.com/chromedp/cdproto v0.0.0-20190412020601-c4267f5c421a/go.mod h1:xquOK9dIGFlLaIGI4c6IyfLI/Gz0LiYYuJtzhsUODgI=
github.com/disintegration/imaging v1.6.0 h1:nVPXRUUQ36Z7MNf0O77UzgnOb1mkMMor7lmJMJXc/mA=
github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiMjXsGrFVK25aOWLq7n+3gh2ls=
github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
github.com/chromedp/cdproto v0.0.0-20180522032958-55db67b53f25/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs=
github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs=
github.com/disintegration/imaging v1.4.2/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
golang.org/x/image v0.0.0-20180403161127-f315e4403028/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20180628062038-cc896f830ced/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=

657
handler.go Normal file
View File

@ -0,0 +1,657 @@
package chromedp
import (
"context"
"encoding/json"
"fmt"
"reflect"
goruntime "runtime"
"strings"
"sync"
"time"
"github.com/mailru/easyjson"
"github.com/chromedp/cdproto"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/css"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/inspector"
"github.com/chromedp/cdproto/log"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/chromedp/client"
)
// TargetHandler manages a Chrome Debugging Protocol target.
type TargetHandler struct {
conn client.Transport
// frames is the set of encountered frames.
frames map[cdp.FrameID]*cdp.Frame
// cur is the current top level frame.
cur *cdp.Frame
// qcmd is the outgoing message queue.
qcmd chan *cdproto.Message
// qres is the incoming command result queue.
qres chan *cdproto.Message
// qevents is the incoming event queue.
qevents chan *cdproto.Message
// detached is closed when the detached event is received.
detached chan *inspector.EventDetached
pageWaitGroup, domWaitGroup *sync.WaitGroup
// last is the last sent message identifier.
last int64
lastm sync.Mutex
// res is the id->result channel map.
res map[int64]chan *cdproto.Message
resrw sync.RWMutex
// logging funcs
logf, debugf, errf func(string, ...interface{})
sync.RWMutex
}
// NewTargetHandler creates a new handler for the specified client target.
func NewTargetHandler(t client.Target, logf, debugf, errf func(string, ...interface{})) (*TargetHandler, error) {
conn, err := client.Dial(t)
if err != nil {
return nil, err
}
return &TargetHandler{
conn: conn,
logf: logf,
debugf: debugf,
errf: errf,
}, nil
}
// Run starts the processing of commands and events of the client target
// provided to NewTargetHandler.
//
// Callers can stop Run by closing the passed context.
func (h *TargetHandler) Run(ctxt context.Context) error {
// reset
h.Lock()
h.frames = make(map[cdp.FrameID]*cdp.Frame)
h.qcmd = make(chan *cdproto.Message)
h.qres = make(chan *cdproto.Message)
h.qevents = make(chan *cdproto.Message)
h.res = make(map[int64]chan *cdproto.Message)
h.detached = make(chan *inspector.EventDetached)
h.pageWaitGroup = new(sync.WaitGroup)
h.domWaitGroup = new(sync.WaitGroup)
h.Unlock()
// run
go h.run(ctxt)
// enable domains
for _, a := range []Action{
log.Enable(),
runtime.Enable(),
//network.Enable(),
inspector.Enable(),
page.Enable(),
dom.Enable(),
css.Enable(),
} {
if err := a.Do(ctxt, h); err != nil {
return fmt.Errorf("unable to execute %s: %v", reflect.TypeOf(a), err)
}
}
h.Lock()
// get page resources
tree, err := page.GetResourceTree().Do(ctxt, h)
if err != nil {
return fmt.Errorf("unable to get resource tree: %v", err)
}
h.frames[tree.Frame.ID] = tree.Frame
h.cur = tree.Frame
for _, c := range tree.ChildFrames {
h.frames[c.Frame.ID] = c.Frame
}
h.Unlock()
h.documentUpdated(ctxt)
return nil
}
// run handles the actual message processing to / from the web socket connection.
func (h *TargetHandler) run(ctxt context.Context) {
defer h.conn.Close()
// add cancel to context
ctxt, cancel := context.WithCancel(ctxt)
defer cancel()
go func() {
defer cancel()
for {
select {
default:
msg, err := h.read()
if err != nil {
return
}
switch {
case msg.Method != "":
h.qevents <- msg
case msg.ID != 0:
h.qres <- msg
default:
h.errf("ignoring malformed incoming message (missing id or method): %#v", msg)
}
case <-h.detached:
// FIXME: should log when detached, and reason
return
case <-ctxt.Done():
return
}
}
}()
// process queues
for {
select {
case ev := <-h.qevents:
err := h.processEvent(ctxt, ev)
if err != nil {
h.errf("could not process event %s: %v", ev.Method, err)
}
case res := <-h.qres:
err := h.processResult(res)
if err != nil {
h.errf("could not process result for message %d: %v", res.ID, err)
}
case cmd := <-h.qcmd:
err := h.processCommand(cmd)
if err != nil {
h.errf("could not process command message %d: %v", cmd.ID, err)
}
case <-ctxt.Done():
return
}
}
}
// read reads a message from the client connection.
func (h *TargetHandler) read() (*cdproto.Message, error) {
// read
buf, err := h.conn.Read()
if err != nil {
return nil, err
}
h.debugf("-> %s", string(buf))
// unmarshal
msg := new(cdproto.Message)
err = json.Unmarshal(buf, msg)
if err != nil {
return nil, err
}
return msg, nil
}
// processEvent processes an incoming event.
func (h *TargetHandler) processEvent(ctxt context.Context, msg *cdproto.Message) error {
if msg == nil {
return ErrChannelClosed
}
// unmarshal
ev, err := cdproto.UnmarshalMessage(msg)
if err != nil {
return err
}
switch e := ev.(type) {
case *inspector.EventDetached:
h.Lock()
defer h.Unlock()
h.detached <- e
return nil
case *dom.EventDocumentUpdated:
h.domWaitGroup.Wait()
go h.documentUpdated(ctxt)
return nil
}
d := msg.Method.Domain()
if d != "Page" && d != "DOM" {
return nil
}
switch d {
case "Page":
h.pageWaitGroup.Add(1)
go h.pageEvent(ctxt, ev)
case "DOM":
h.domWaitGroup.Add(1)
go h.domEvent(ctxt, ev)
}
return nil
}
// documentUpdated handles the document updated event, retrieving the document
// root for the root frame.
func (h *TargetHandler) documentUpdated(ctxt context.Context) {
f, err := h.WaitFrame(ctxt, cdp.EmptyFrameID)
if err != nil {
h.errf("could not get current frame: %v", err)
return
}
f.Lock()
defer f.Unlock()
// invalidate nodes
if f.Root != nil {
close(f.Root.Invalidated)
}
f.Nodes = make(map[cdp.NodeID]*cdp.Node)
f.Root, err = dom.GetDocument().WithPierce(true).Do(ctxt, h)
if err != nil {
h.errf("could not retrieve document root for %s: %v", f.ID, err)
return
}
f.Root.Invalidated = make(chan struct{})
walk(f.Nodes, f.Root)
}
// processResult processes an incoming command result.
func (h *TargetHandler) processResult(msg *cdproto.Message) error {
h.resrw.RLock()
defer h.resrw.RUnlock()
ch, ok := h.res[msg.ID]
if !ok {
return fmt.Errorf("id %d not present in res map", msg.ID)
}
defer close(ch)
ch <- msg
return nil
}
// processCommand writes a command to the client connection.
func (h *TargetHandler) processCommand(cmd *cdproto.Message) error {
// marshal
buf, err := json.Marshal(cmd)
if err != nil {
return err
}
h.debugf("<- %s", string(buf))
return h.conn.Write(buf)
}
// emptyObj is an empty JSON object message.
var emptyObj = easyjson.RawMessage([]byte(`{}`))
// Execute executes commandType against the endpoint passed to Run, using the
// provided context and params, decoding the result of the command to res.
func (h *TargetHandler) Execute(ctxt context.Context, methodType string, params json.Marshaler, res json.Unmarshaler) error {
var paramsBuf easyjson.RawMessage
if params == nil {
paramsBuf = emptyObj
} else {
var err error
paramsBuf, err = json.Marshal(params)
if err != nil {
return err
}
}
id := h.next()
// save channel
ch := make(chan *cdproto.Message, 1)
h.resrw.Lock()
h.res[id] = ch
h.resrw.Unlock()
// queue message
h.qcmd <- &cdproto.Message{
ID: id,
Method: cdproto.MethodType(methodType),
Params: paramsBuf,
}
errch := make(chan error, 1)
go func() {
defer close(errch)
select {
case msg := <-ch:
switch {
case msg == nil:
errch <- ErrChannelClosed
case msg.Error != nil:
errch <- msg.Error
case res != nil:
errch <- json.Unmarshal(msg.Result, res)
}
case <-ctxt.Done():
errch <- ctxt.Err()
}
h.resrw.Lock()
defer h.resrw.Unlock()
delete(h.res, id)
}()
return <-errch
}
// next returns the next message id.
func (h *TargetHandler) next() int64 {
h.lastm.Lock()
defer h.lastm.Unlock()
h.last++
return h.last
}
// GetRoot returns the current top level frame's root document node.
func (h *TargetHandler) GetRoot(ctxt context.Context) (*cdp.Node, error) {
var root *cdp.Node
for {
var cur *cdp.Frame
select {
default:
h.RLock()
cur = h.cur
if cur != nil {
cur.RLock()
root = cur.Root
cur.RUnlock()
}
h.RUnlock()
if cur != nil && root != nil {
return root, nil
}
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return nil, ctxt.Err()
}
}
}
// SetActive sets the currently active frame after a successful navigation.
func (h *TargetHandler) SetActive(ctxt context.Context, id cdp.FrameID) error {
var err error
// get frame
f, err := h.WaitFrame(ctxt, id)
if err != nil {
return err
}
h.Lock()
defer h.Unlock()
h.cur = f
return nil
}
// WaitFrame waits for a frame to be loaded using the provided context.
func (h *TargetHandler) WaitFrame(ctxt context.Context, id cdp.FrameID) (*cdp.Frame, error) {
// TODO: fix this
timeout := time.After(10 * time.Second)
for {
select {
default:
var f *cdp.Frame
var ok bool
h.RLock()
if id == cdp.EmptyFrameID {
f, ok = h.cur, h.cur != nil
} else {
f, ok = h.frames[id]
}
h.RUnlock()
if ok {
return f, nil
}
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return nil, ctxt.Err()
case <-timeout:
return nil, fmt.Errorf("timeout waiting for frame `%s`", id)
}
}
}
// WaitNode waits for a node to be loaded using the provided context.
func (h *TargetHandler) WaitNode(ctxt context.Context, f *cdp.Frame, id cdp.NodeID) (*cdp.Node, error) {
// TODO: fix this
timeout := time.After(10 * time.Second)
for {
select {
default:
var n *cdp.Node
var ok bool
f.RLock()
n, ok = f.Nodes[id]
f.RUnlock()
if n != nil && ok {
return n, nil
}
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return nil, ctxt.Err()
case <-timeout:
return nil, fmt.Errorf("timeout waiting for node `%d`", id)
}
}
}
// pageEvent handles incoming page events.
func (h *TargetHandler) pageEvent(ctxt context.Context, ev interface{}) {
defer h.pageWaitGroup.Done()
var id cdp.FrameID
var op frameOp
switch e := ev.(type) {
case *page.EventFrameNavigated:
h.Lock()
h.frames[e.Frame.ID] = e.Frame
if h.cur != nil && h.cur.ID == e.Frame.ID {
h.cur = e.Frame
}
h.Unlock()
return
case *page.EventFrameAttached:
id, op = e.FrameID, frameAttached(e.ParentFrameID)
case *page.EventFrameDetached:
id, op = e.FrameID, frameDetached
case *page.EventFrameStartedLoading:
id, op = e.FrameID, frameStartedLoading
case *page.EventFrameStoppedLoading:
id, op = e.FrameID, frameStoppedLoading
case *page.EventFrameScheduledNavigation:
id, op = e.FrameID, frameScheduledNavigation
case *page.EventFrameClearedScheduledNavigation:
id, op = e.FrameID, frameClearedScheduledNavigation
// ignored events
case *page.EventDomContentEventFired:
return
case *page.EventLoadEventFired:
return
case *page.EventFrameResized:
return
case *page.EventLifecycleEvent:
return
default:
h.errf("unhandled page event %s", reflect.TypeOf(ev))
return
}
f, err := h.WaitFrame(ctxt, id)
if err != nil {
h.errf("could not get frame %s: %v", id, err)
return
}
h.Lock()
defer h.Unlock()
f.Lock()
defer f.Unlock()
op(f)
}
// domEvent handles incoming DOM events.
func (h *TargetHandler) domEvent(ctxt context.Context, ev interface{}) {
defer h.domWaitGroup.Done()
// wait current frame
f, err := h.WaitFrame(ctxt, cdp.EmptyFrameID)
if err != nil {
h.errf("could not process DOM event %s: %v", reflect.TypeOf(ev), err)
return
}
var id cdp.NodeID
var op nodeOp
switch e := ev.(type) {
case *dom.EventSetChildNodes:
id, op = e.ParentID, setChildNodes(f.Nodes, e.Nodes)
case *dom.EventAttributeModified:
id, op = e.NodeID, attributeModified(e.Name, e.Value)
case *dom.EventAttributeRemoved:
id, op = e.NodeID, attributeRemoved(e.Name)
case *dom.EventInlineStyleInvalidated:
if len(e.NodeIds) == 0 {
return
}
id, op = e.NodeIds[0], inlineStyleInvalidated(e.NodeIds[1:])
case *dom.EventCharacterDataModified:
id, op = e.NodeID, characterDataModified(e.CharacterData)
case *dom.EventChildNodeCountUpdated:
id, op = e.NodeID, childNodeCountUpdated(e.ChildNodeCount)
case *dom.EventChildNodeInserted:
if e.PreviousNodeID != cdp.EmptyNodeID {
_, err = h.WaitNode(ctxt, f, e.PreviousNodeID)
if err != nil {
return
}
}
id, op = e.ParentNodeID, childNodeInserted(f.Nodes, e.PreviousNodeID, e.Node)
case *dom.EventChildNodeRemoved:
id, op = e.ParentNodeID, childNodeRemoved(f.Nodes, e.NodeID)
case *dom.EventShadowRootPushed:
id, op = e.HostID, shadowRootPushed(f.Nodes, e.Root)
case *dom.EventShadowRootPopped:
id, op = e.HostID, shadowRootPopped(f.Nodes, e.RootID)
case *dom.EventPseudoElementAdded:
id, op = e.ParentID, pseudoElementAdded(f.Nodes, e.PseudoElement)
case *dom.EventPseudoElementRemoved:
id, op = e.ParentID, pseudoElementRemoved(f.Nodes, e.PseudoElementID)
case *dom.EventDistributedNodesUpdated:
id, op = e.InsertionPointID, distributedNodesUpdated(e.DistributedNodes)
default:
h.errf("unhandled node event %s", reflect.TypeOf(ev))
return
}
// retrieve node
n, err := h.WaitNode(ctxt, f, id)
if err != nil {
s := strings.TrimSuffix(goruntime.FuncForPC(reflect.ValueOf(op).Pointer()).Name(), ".func1")
i := strings.LastIndex(s, ".")
if i != -1 {
s = s[i+1:]
}
h.errf("could not perform (%s) operation on node %d (wait node): %v", s, id, err)
return
}
h.Lock()
defer h.Unlock()
f.Lock()
defer f.Unlock()
op(n)
}

36
input.go
View File

@ -3,12 +3,13 @@ package chromedp
import (
"context"
"fmt"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/input"
"git.loafle.net/commons_go/chromedp/kb"
"github.com/chromedp/chromedp/kb"
)
// MouseAction is a mouse action.
@ -26,7 +27,7 @@ func MouseAction(typ input.MouseType, x, y int64, opts ...MouseOption) Action {
// MouseClickXY sends a left mouse button click (ie, mousePressed and
// mouseReleased event) at the X, Y location.
func MouseClickXY(x, y int64, opts ...MouseOption) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
me := &input.DispatchMouseEventParams{
Type: input.MousePressed,
X: float64(x),
@ -40,12 +41,13 @@ func MouseClickXY(x, y int64, opts ...MouseOption) Action {
me = o(me)
}
if err := me.Do(ctx, h); err != nil {
err := me.Do(ctxt, h)
if err != nil {
return err
}
me.Type = input.MouseReleased
return me.Do(ctx, h)
return me.Do(ctxt, h)
})
}
@ -55,14 +57,16 @@ func MouseClickXY(x, y int64, opts ...MouseOption) Action {
// Note that the window will be scrolled if the node is not within the window's
// viewport.
func MouseClickNode(n *cdp.Node, opts ...MouseOption) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
var pos []int
err := EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, n.FullXPath()), &pos).Do(ctx, h)
err = EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, n.FullXPath()), &pos).Do(ctxt, h)
if err != nil {
return err
}
box, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctx, h)
box, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctxt, h)
if err != nil {
return err
}
@ -80,7 +84,7 @@ func MouseClickNode(n *cdp.Node, opts ...MouseOption) Action {
x /= int64(c / 2)
y /= int64(c / 2)
return MouseClickXY(x, y, opts...).Do(ctx, h)
return MouseClickXY(x, y, opts...).Do(ctxt, h)
})
}
@ -149,13 +153,19 @@ func ClickCount(n int) MouseOption {
// Please see the chromedp/kb package for implementation details and the list
// of well-known keys.
func KeyAction(keys string, opts ...KeyOption) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
for _, r := range keys {
for _, k := range kb.Encode(r) {
if err := k.Do(ctx, h); err != nil {
err = k.Do(ctxt, h)
if err != nil {
return err
}
}
// TODO: move to context
time.Sleep(5 * time.Millisecond)
}
return nil
@ -164,13 +174,13 @@ func KeyAction(keys string, opts ...KeyOption) Action {
// KeyActionNode dispatches a key event on a node.
func KeyActionNode(n *cdp.Node, keys string, opts ...KeyOption) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
err := dom.Focus().WithNodeID(n.NodeID).Do(ctx, h)
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
err := dom.Focus().WithNodeID(n.NodeID).Do(ctxt, h)
if err != nil {
return err
}
return KeyAction(keys, opts...).Do(ctx, h)
return KeyAction(keys, opts...).Do(ctxt, h)
})
}

148
input_test.go
View File

@ -4,28 +4,35 @@ import (
"fmt"
"strconv"
"testing"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/input"
)
const (
// inViewportJS is a javascript snippet that will get the specified node
// position relative to the viewport and returns true if the specified node
// is within the window's viewport.
const inViewportJS = `(function(a) {
inViewportJS = `(function(a) {
var r = a[0].getBoundingClientRect();
return r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth;
})($x('%s'))`
)
func TestMouseClickXY(t *testing.T) {
var err error
t.Parallel()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
c := testAllocate(t, "input.html")
defer c.Release()
if err := Run(ctx, WaitVisible(`#input1`, ByID)); err != nil {
err = c.Run(defaultContext, Sleep(100*time.Millisecond))
if err != nil {
t.Fatal(err)
}
tests := []struct {
x, y int64
}{
@ -36,14 +43,18 @@ func TestMouseClickXY(t *testing.T) {
}
for i, test := range tests {
var xstr, ystr string
if err := Run(ctx,
MouseClickXY(test.x, test.y),
Value("#input1", &xstr, ByID),
); err != nil {
err = c.Run(defaultContext, MouseClickXY(test.x, test.y))
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
time.Sleep(50 * time.Millisecond)
var xstr, ystr string
err = c.Run(defaultContext, Value("#input1", &xstr, ByID))
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
x, err := strconv.ParseInt(xstr, 10, 64)
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
@ -51,10 +62,11 @@ func TestMouseClickXY(t *testing.T) {
if x != test.x {
t.Fatalf("test %d expected x to be: %d, got: %d", i, test.x, x)
}
if err := Run(ctx, Value("#input2", &ystr, ByID)); err != nil {
err = c.Run(defaultContext, Value("#input2", &ystr, ByID))
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
y, err := strconv.ParseInt(ystr, 10, 64)
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
@ -66,8 +78,6 @@ func TestMouseClickXY(t *testing.T) {
}
func TestMouseClickNode(t *testing.T) {
t.Parallel()
tests := []struct {
sel, exp string
opt MouseOption
@ -76,34 +86,40 @@ func TestMouseClickNode(t *testing.T) {
{"button2", "foo", ButtonType(input.ButtonNone), ByID},
{"button2", "bar", ButtonType(input.ButtonLeft), ByID},
{"button2", "bar-middle", ButtonType(input.ButtonMiddle), ByID},
{"input3", "foo", ButtonModifiers(input.ModifierNone), ByID},
{"input3", "bar-right", ButtonType(input.ButtonRight), ByID},
{"input3", "bar-right", ButtonModifiers(input.ModifierNone), ByID},
{"input3", "bar-right", Button("right"), ByID},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
test := test
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
c := testAllocate(t, "input.html")
defer c.Release()
var err error
var nodes []*cdp.Node
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
var value string
if err := Run(ctx,
MouseClickNode(nodes[0], test.opt),
Value("#input3", &value, ByID),
); err != nil {
err = c.Run(defaultContext, MouseClickNode(nodes[0], test.opt))
if err != nil {
t.Fatalf("got error: %v", err)
}
time.Sleep(50 * time.Millisecond)
var value string
err = c.Run(defaultContext, Value("#input3", &value, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}
@ -112,8 +128,6 @@ func TestMouseClickNode(t *testing.T) {
}
func TestMouseClickOffscreenNode(t *testing.T) {
t.Parallel()
tests := []struct {
sel string
exp int
@ -125,42 +139,45 @@ func TestMouseClickOffscreenNode(t *testing.T) {
}
for i, test := range tests {
t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
test := test
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
c := testAllocate(t, "input.html")
defer c.Release()
var err error
var nodes []*cdp.Node
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
var ok bool
if err := Run(ctx, EvaluateAsDevTools(fmt.Sprintf(inViewportJS, nodes[0].FullXPath()), &ok)); err != nil {
err = c.Run(defaultContext, EvaluateAsDevTools(fmt.Sprintf(inViewportJS, nodes[0].FullXPath()), &ok))
if err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected node to be offscreen")
}
for i := test.exp; i > 0; i-- {
if err := Run(ctx, MouseClickNode(nodes[0])); err != nil {
err = c.Run(defaultContext, MouseClickNode(nodes[0]))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
time.Sleep(100 * time.Millisecond)
var value int
if err := Run(ctx, Evaluate("window.document.test_i", &value)); err != nil {
err = c.Run(defaultContext, Evaluate("window.document.test_i", &value))
if err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %d, got: %d", test.exp, value)
}
@ -169,8 +186,6 @@ func TestMouseClickOffscreenNode(t *testing.T) {
}
func TestKeyAction(t *testing.T) {
t.Parallel()
tests := []struct {
sel, exp string
by QueryOption
@ -184,33 +199,37 @@ func TestKeyAction(t *testing.T) {
}
for i, test := range tests {
t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
test := test
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
c := testAllocate(t, "input.html")
defer c.Release()
var err error
var nodes []*cdp.Node
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
if err := Run(ctx,
Focus(test.sel, test.by),
KeyAction(test.exp),
); err != nil {
err = c.Run(defaultContext, Focus(test.sel, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, KeyAction(test.exp))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}
@ -219,8 +238,6 @@ func TestKeyAction(t *testing.T) {
}
func TestKeyActionNode(t *testing.T) {
t.Parallel()
tests := []struct {
sel, exp string
by QueryOption
@ -234,29 +251,32 @@ func TestKeyActionNode(t *testing.T) {
}
for i, test := range tests {
t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
test := test
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
c := testAllocate(t, "input.html")
defer c.Release()
var err error
var nodes []*cdp.Node
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
var value string
if err := Run(ctx,
KeyActionNode(nodes[0], test.exp),
Value(test.sel, &value, test.by),
); err != nil {
err = c.Run(defaultContext, KeyActionNode(nodes[0], test.exp))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}

24
kb/gen.go
View File

@ -16,7 +16,7 @@ import (
"strconv"
"strings"
"git.loafle.net/commons_go/chromedp/kb"
"github.com/chromedp/chromedp/kb"
)
var (
@ -72,6 +72,8 @@ func main() {
}
func run() error {
var err error
// special characters
keys := map[rune]kb.Key{
'\b': {"Backspace", "Backspace", "", "", int64('\b'), int64('\b'), false, false},
@ -80,7 +82,8 @@ func run() error {
}
// load keys
if err := loadKeys(keys); err != nil {
err = loadKeys(keys)
if err != nil {
return err
}
@ -91,19 +94,24 @@ func run() error {
}
// output
if err := ioutil.WriteFile(*flagOut,
err = ioutil.WriteFile(
*flagOut,
[]byte(fmt.Sprintf(hdr, *flagPkg, string(constBuf), string(mapBuf))),
0644); err != nil {
0644,
)
if err != nil {
return err
}
// format
if err := exec.Command("goimports", "-w", *flagOut).Run(); err != nil {
err = exec.Command("goimports", "-w", *flagOut).Run()
if err != nil {
return err
}
// format
if err := exec.Command("gofmt", "-s", "-w", *flagOut).Run(); err != nil {
err = exec.Command("gofmt", "-s", "-w", *flagOut).Run()
if err != nil {
return err
}
@ -112,6 +120,8 @@ func run() error {
// loadKeys loads the dom key definitions from the chromium source tree.
func loadKeys(keys map[rune]kb.Key) error {
var err error
// load key converter data
keycodeConverterMap, err := loadKeycodeConverterData()
if err != nil {
@ -434,6 +444,8 @@ var defineRE = regexp.MustCompile(`(?m)^#define\s+(.+?)\s+([0-9A-Fx]+)`)
// loadPosixWinKeyboardCodes loads the native and windows keyboard scan codes
// mapped to the DOM key.
func loadPosixWinKeyboardCodes() (map[string][]int64, error) {
var err error
lookup := map[string]string{
// mac alias
"VKEY_LWIN": "0x5B",

37
nav.go
View File

@ -10,9 +10,18 @@ import (
// Navigate navigates the current frame.
func Navigate(urlstr string) Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
_, _, _, err := page.Navigate(urlstr).Do(ctx, h)
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
th, ok := h.(*TargetHandler)
if !ok {
return ErrInvalidHandler
}
frameID, _, _, err := page.Navigate(urlstr).Do(ctxt, th)
if err != nil {
return err
}
return th.SetActive(ctxt, frameID)
})
}
@ -23,9 +32,9 @@ func NavigationEntries(currentIndex *int64, entries *[]*page.NavigationEntry) Ac
panic("currentIndex and entries cannot be nil")
}
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
*currentIndex, *entries, err = page.GetNavigationHistory().Do(ctx, h)
*currentIndex, *entries, err = page.GetNavigationHistory().Do(ctxt, h)
return err
})
}
@ -38,8 +47,8 @@ func NavigateToHistoryEntry(entryID int64) Action {
// NavigateBack navigates the current frame backwards in its history.
func NavigateBack() Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
cur, entries, err := page.GetNavigationHistory().Do(ctx, h)
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
cur, entries, err := page.GetNavigationHistory().Do(ctxt, h)
if err != nil {
return err
}
@ -48,14 +57,14 @@ func NavigateBack() Action {
return errors.New("invalid navigation entry")
}
return page.NavigateToHistoryEntry(entries[cur-1].ID).Do(ctx, h)
return page.NavigateToHistoryEntry(entries[cur-1].ID).Do(ctxt, h)
})
}
// NavigateForward navigates the current frame forwards in its history.
func NavigateForward() Action {
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
cur, entries, err := page.GetNavigationHistory().Do(ctx, h)
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
cur, entries, err := page.GetNavigationHistory().Do(ctxt, h)
if err != nil {
return err
}
@ -64,7 +73,7 @@ func NavigateForward() Action {
return errors.New("invalid navigation entry")
}
return page.NavigateToHistoryEntry(entries[cur+1].ID).Do(ctx, h)
return page.NavigateToHistoryEntry(entries[cur+1].ID).Do(ctxt, h)
})
}
@ -86,9 +95,9 @@ func CaptureScreenshot(res *[]byte) Action {
panic("res cannot be nil")
}
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
*res, err = page.CaptureScreenshot().Do(ctx, h)
*res, err = page.CaptureScreenshot().Do(ctxt, h)
return err
})
}
@ -99,9 +108,9 @@ func CaptureScreenshot(res *[]byte) Action {
panic("id cannot be nil")
}
return ActionFunc(func(ctx context.Context, h cdp.Executor) error {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
*id, err = page.AddScriptToEvaluateOnLoad(source).Do(ctx, h)
*id, err = page.AddScriptToEvaluateOnLoad(source).Do(ctxt, h)
return err
})
}

427
nav_test.go
View File

@ -1,43 +1,47 @@
package chromedp
import (
"bytes"
"fmt"
"image"
_ "image/png"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/chromedp/cdproto/emulation"
"github.com/chromedp/cdproto/page"
)
func TestNavigate(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
var err error
var urlstr string
if err := Run(ctx,
WaitVisible(`#icon-brankas`, ByID),
Location(&urlstr),
); err != nil {
c := testAllocate(t, "")
defer c.Release()
expurl, exptitle := testdataDir+"/image.html", "this is title"
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(urlstr, "image.html") {
err = c.Run(defaultContext, WaitVisible(`#icon-brankas`, ByID))
if err != nil {
t.Fatal(err)
}
var urlstr string
err = c.Run(defaultContext, Location(&urlstr))
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(urlstr, expurl) {
t.Errorf("expected to be on image.html, at: %s", urlstr)
}
var title string
if err := Run(ctx, Title(&title)); err != nil {
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
exptitle := "this is title"
if title != exptitle {
t.Errorf("expected title to contain google, instead title is: %s", title)
}
@ -46,19 +50,21 @@ func TestNavigate(t *testing.T) {
func TestNavigationEntries(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "")
defer cancel()
var err error
tests := []struct {
file, waitID string
}{
{"form.html", "#form"},
{"image.html", "#icon-brankas"},
c := testAllocate(t, "")
defer c.Release()
tests := []string{
"form.html",
"image.html",
}
var entries []*page.NavigationEntry
var index int64
if err := Run(ctx, NavigationEntries(&index, &entries)); err != nil {
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
t.Fatal(err)
}
@ -70,19 +76,24 @@ func TestNavigationEntries(t *testing.T) {
}
expIdx, expEntries := 1, 2
for i, test := range tests {
if err := Run(ctx,
Navigate(testdataDir+"/"+test.file),
WaitVisible(test.waitID, ByID),
NavigationEntries(&index, &entries),
); err != nil {
for i, url := range tests {
err = c.Run(defaultContext, Navigate(testdataDir+"/"+url))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
t.Fatal(err)
}
if len(entries) != expEntries {
t.Errorf("test %d expected to have %d navigation entry: got %d", i, expEntries, len(entries))
}
if want := int64(i + 1); index != want {
t.Errorf("test %d expected navigation index is %d, got: %d", i, want, index)
if index != int64(i+1) {
t.Errorf("test %d expected navigation index is %d, got: %d", i, i, index)
}
expIdx++
@ -93,27 +104,42 @@ func TestNavigationEntries(t *testing.T) {
func TestNavigateToHistoryEntry(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
var err error
c := testAllocate(t, "")
defer c.Release()
var entries []*page.NavigationEntry
var index int64
if err := Run(ctx,
WaitVisible(`#icon-brankas`, ByID), // for image.html
NavigationEntries(&index, &entries),
Navigate(testdataDir+"/form.html"),
WaitVisible(`#form`, ByID), // for form.html
); err != nil {
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateToHistoryEntry(entries[index].ID))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
if err := Run(ctx,
NavigateToHistoryEntry(entries[index].ID),
WaitVisible(`#icon-brankas`, ByID), // for image.html
Title(&title),
); err != nil {
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
if title != entries[index].Title {
@ -124,24 +150,43 @@ func TestNavigateToHistoryEntry(t *testing.T) {
func TestNavigateBack(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
var err error
var title, exptitle string
if err := Run(ctx,
WaitVisible(`#form`, ByID), // for form.html
Title(&exptitle),
c := testAllocate(t, "")
defer c.Release()
Navigate(testdataDir+"/image.html"),
WaitVisible(`#icon-brankas`, ByID), // for image.html
NavigateBack(),
WaitVisible(`#form`, ByID), // for form.html
Title(&title),
); err != nil {
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateBack())
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -150,27 +195,50 @@ func TestNavigateBack(t *testing.T) {
func TestNavigateForward(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
var err error
var title, exptitle string
if err := Run(ctx,
WaitVisible(`#form`, ByID), // for form.html
c := testAllocate(t, "")
defer c.Release()
Navigate(testdataDir+"/image.html"),
WaitVisible(`#icon-brankas`, ByID), // for image.html
Title(&exptitle),
NavigateBack(),
WaitVisible(`#form`, ByID), // for form.html
NavigateForward(),
WaitVisible(`#icon-brankas`, ByID), // for image.html
Title(&title),
); err != nil {
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, NavigateBack())
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateForward())
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -179,9 +247,18 @@ func TestNavigateForward(t *testing.T) {
func TestStop(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
if err := Run(ctx, Stop()); err != nil {
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Stop())
if err != nil {
t.Fatal(err)
}
}
@ -189,38 +266,36 @@ func TestStop(t *testing.T) {
func TestReload(t *testing.T) {
t.Parallel()
count := 0
// create test server
mux := http.NewServeMux()
mux.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) {
fmt.Fprintf(res, `<html>
<head>
<title>Title</title>
</head>
<body>
<div id="count%d"></div>
</body></html`, count)
count++
})
s := httptest.NewServer(mux)
defer s.Close()
var err error
ctx, cancel := testAllocate(t, "")
defer cancel()
c := testAllocate(t, "")
defer c.Release()
var title, exptitle string
if err := Run(ctx,
Navigate(s.URL),
WaitReady(`#count0`, ByID),
Title(&exptitle),
Reload(),
WaitReady(`#count1`, ByID),
Title(&title),
); err != nil {
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Reload())
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -229,47 +304,51 @@ func TestReload(t *testing.T) {
func TestCaptureScreenshot(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
var err error
// set the viewport size, to know what screenshot size to expect
width, height := 650, 450
var buf []byte
if err := Run(ctx,
emulation.SetDeviceMetricsOverride(int64(width), int64(height), 1.0, false),
WaitVisible(`#icon-brankas`, ByID), // for image.html
CaptureScreenshot(&buf),
); err != nil {
t.Fatal(err)
}
c := testAllocate(t, "")
defer c.Release()
config, format, err := image.DecodeConfig(bytes.NewReader(buf))
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
t.Fatal(err)
}
if want := "png"; format != want {
t.Fatalf("expected format to be %q, got %q", want, format)
time.Sleep(50 * time.Millisecond)
var buf []byte
err = c.Run(defaultContext, CaptureScreenshot(&buf))
if err != nil {
t.Fatal(err)
}
if config.Width != width || config.Height != height {
t.Fatalf("expected dimensions to be %d*%d, got %d*%d",
width, height, config.Width, config.Height)
if len(buf) == 0 {
t.Fatal("failed to capture screenshot")
}
//TODO: test image
}
/*func TestAddOnLoadScript(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "")
defer cancel()
var err error
c := testAllocate(t, "")
defer c.Release()
var scriptID page.ScriptIdentifier
if err := Run(ctx,
AddOnLoadScript(`window.alert("TEST")`, &scriptID),
Navigate(testdataDir+"/form.html"),
); err != nil {
err = c.Run(defaultContext, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
if scriptID == "" {
t.Fatal("got empty script ID")
}
@ -279,40 +358,57 @@ func TestCaptureScreenshot(t *testing.T) {
func TestRemoveOnLoadScript(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "")
defer cancel()
var err error
c := testAllocate(t, "")
defer c.Release()
var scriptID page.ScriptIdentifier
if err := Run(ctx, AddOnLoadScript(`window.alert("TEST")`, &scriptID)); err != nil {
err = c.Run(defaultContext, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
if err != nil {
t.Fatal(err)
}
if scriptID == "" {
t.Fatal("got empty script ID")
}
if err := Run(ctx,
RemoveOnLoadScript(scriptID),
Navigate(testdataDir+"/form.html"),
); err != nil {
err = c.Run(defaultContext, RemoveOnLoadScript(scriptID))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
}*/
func TestLocation(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
var err error
expurl := testdataDir + "/form.html"
var urlstr string
if err := Run(ctx,
WaitVisible(`#form`, ByID), // for form.html
Location(&urlstr),
); err != nil {
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(urlstr, "form.html") {
time.Sleep(50 * time.Millisecond)
var urlstr string
err = c.Run(defaultContext, Location(&urlstr))
if err != nil {
t.Fatal(err)
}
if urlstr != expurl {
t.Fatalf("expected to be on form.html, got: %s", urlstr)
}
}
@ -320,35 +416,26 @@ func TestLocation(t *testing.T) {
func TestTitle(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
var err error
expurl, exptitle := testdataDir+"/image.html", "this is title"
var title string
if err := Run(ctx,
WaitVisible(`#icon-brankas`, ByID), // for image.html
Title(&title),
); err != nil {
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
t.Fatal(err)
}
exptitle := "this is title"
if title != exptitle {
t.Fatalf("expected title to be %s, got: %s", exptitle, title)
}
}
func TestLoadIframe(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "iframe.html")
defer cancel()
if err := Run(ctx, Tasks{
// TODO: remove the sleep once we have better support for
// iframes.
Sleep(10 * time.Millisecond),
// WaitVisible(`#form`, ByID), // for the nested form.html
}); err != nil {
t.Fatal(err)
}
}

202
pool.go Normal file
View File

@ -0,0 +1,202 @@
package chromedp
import (
"context"
"fmt"
"log"
"sync"
"github.com/chromedp/chromedp/runner"
)
// Pool manages a pool of running Chrome processes.
type Pool struct {
// start is the start port.
start int
// end is the end port.
end int
// res are the running chrome resources.
res map[int]*Res
// logging funcs
logf, debugf, errf func(string, ...interface{})
rw sync.RWMutex
}
// NewPool creates a new Chrome runner pool.
func NewPool(opts ...PoolOption) (*Pool, error) {
p := &Pool{
start: DefaultPoolStartPort,
end: DefaultPoolEndPort,
res: make(map[int]*Res),
logf: log.Printf,
debugf: func(string, ...interface{}) {},
}
// apply opts
for _, o := range opts {
if err := o(p); err != nil {
return nil, err
}
}
if p.errf == nil {
p.errf = func(s string, v ...interface{}) {
p.logf("ERROR: "+s, v...)
}
}
return p, nil
}
// Shutdown releases all the pool resources.
func (p *Pool) Shutdown() error {
p.rw.Lock()
defer p.rw.Unlock()
for _, r := range p.res {
r.cancel()
}
return nil
}
// Allocate creates a new process runner and returns it.
func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption) (*Res, error) {
var err error
r := p.next(ctxt)
p.debugf("pool allocating %d", r.port)
// create runner
r.r, err = runner.New(append([]runner.CommandLineOption{
runner.HeadlessPathPort("", r.port),
}, opts...)...)
if err != nil {
defer r.Release()
p.errf("pool could not allocate runner on port %d: %v", r.port, err)
return nil, err
}
// start runner
err = r.r.Start(r.ctxt)
if err != nil {
defer r.Release()
p.errf("pool could not start runner on port %d: %v", r.port, err)
return nil, err
}
// setup cdp
r.c, err = New(
r.ctxt, WithRunner(r.r),
WithLogf(p.logf), WithDebugf(p.debugf), WithErrorf(p.errf),
)
if err != nil {
defer r.Release()
p.errf("pool could not connect to %d: %v", r.port, err)
return nil, err
}
return r, nil
}
// next returns the next available res.
func (p *Pool) next(ctxt context.Context) *Res {
p.rw.Lock()
defer p.rw.Unlock()
var found bool
var i int
for i = p.start; i < p.end; i++ {
if _, ok := p.res[i]; !ok {
found = true
break
}
}
if !found {
panic("no ports available")
}
r := &Res{
p: p,
port: i,
}
r.ctxt, r.cancel = context.WithCancel(ctxt)
p.res[i] = r
return r
}
// Res is a pool resource.
type Res struct {
p *Pool
ctxt context.Context
cancel func()
port int
r *runner.Runner
c *CDP
}
// Release releases the pool resource.
func (r *Res) Release() error {
r.cancel()
var err error
if r.c != nil {
err = r.c.Wait()
}
defer r.p.debugf("pool released %d", r.port)
r.p.rw.Lock()
defer r.p.rw.Unlock()
delete(r.p.res, r.port)
return err
}
// Port returns the allocated port for the pool resource.
func (r *Res) Port() int {
return r.port
}
// URL returns a formatted URL for the pool resource.
func (r *Res) URL() string {
return fmt.Sprintf("http://localhost:%d/json", r.port)
}
// CDP returns the actual CDP instance.
func (r *Res) CDP() *CDP {
return r.c
}
// Run runs an action.
func (r *Res) Run(ctxt context.Context, a Action) error {
return r.c.Run(ctxt, a)
}
// PoolOption is a pool option.
type PoolOption func(*Pool) error
// PortRange is a pool option to set the port range to use.
func PortRange(start, end int) PoolOption {
return func(p *Pool) error {
p.start = start
p.end = end
return nil
}
}
// PoolLog is a pool option to set the logging to use for the pool.
func PoolLog(logf, debugf, errf func(string, ...interface{})) PoolOption {
return func(p *Pool) error {
p.logf, p.debugf, p.errf = logf, debugf, errf
return nil
}
}

99
query.go
View File

@ -25,7 +25,7 @@ func Nodes(sel interface{}, nodes *[]*cdp.Node, opts ...QueryOption) Action {
panic("nodes cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, n ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, n ...*cdp.Node) error {
*nodes = n
return nil
}, opts...)
@ -37,7 +37,7 @@ func NodeIDs(sel interface{}, ids *[]cdp.NodeID, opts ...QueryOption) Action {
panic("nodes cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
nodeIDs := make([]cdp.NodeID, len(nodes))
for i, n := range nodes {
nodeIDs[i] = n.NodeID
@ -51,24 +51,24 @@ func NodeIDs(sel interface{}, ids *[]cdp.NodeID, opts ...QueryOption) Action {
// Focus focuses the first node matching the selector.
func Focus(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return dom.Focus().WithNodeID(nodes[0].NodeID).Do(ctx, h)
return dom.Focus().WithNodeID(nodes[0].NodeID).Do(ctxt, h)
}, opts...)
}
// Blur unfocuses (blurs) the first node matching the selector.
func Blur(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var res bool
err := EvaluateAsDevTools(fmt.Sprintf(blurJS, nodes[0].FullXPath()), &res).Do(ctx, h)
err := EvaluateAsDevTools(fmt.Sprintf(blurJS, nodes[0].FullXPath()), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -87,12 +87,12 @@ func Dimensions(sel interface{}, model **dom.BoxModel, opts ...QueryOption) Acti
if model == nil {
panic("model cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var err error
*model, err = dom.GetBoxModel().WithNodeID(nodes[0].NodeID).Do(ctx, h)
*model, err = dom.GetBoxModel().WithNodeID(nodes[0].NodeID).Do(ctxt, h)
return err
}, opts...)
}
@ -103,18 +103,18 @@ func Text(sel interface{}, text *string, opts ...QueryOption) Action {
panic("text cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return EvaluateAsDevTools(fmt.Sprintf(textJS, nodes[0].FullXPath()), text).Do(ctx, h)
return EvaluateAsDevTools(fmt.Sprintf(textJS, nodes[0].FullXPath()), text).Do(ctxt, h)
}, opts...)
}
// Clear clears the values of any input/textarea nodes matching the selector.
func Clear(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
@ -154,7 +154,7 @@ func Clear(sel interface{}, opts ...QueryOption) Action {
a = dom.SetNodeValue(textID, "")
}
errs[i] = a.Do(ctx, h)
errs[i] = a.Do(ctxt, h)
}(i, n)
}
wg.Wait()
@ -190,7 +190,7 @@ func Attributes(sel interface{}, attributes *map[string]string, opts ...QueryOpt
panic("attributes cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
@ -219,7 +219,7 @@ func AttributesAll(sel interface{}, attributes *[]map[string]string, opts ...Que
panic("attributes cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
@ -243,7 +243,7 @@ func AttributesAll(sel interface{}, attributes *[]map[string]string, opts ...Que
// SetAttributes sets the element attributes for the first node matching the
// selector.
func SetAttributes(sel interface{}, attributes map[string]string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return errors.New("expected at least one element")
}
@ -254,7 +254,7 @@ func SetAttributes(sel interface{}, attributes map[string]string, opts ...QueryO
i++
}
return dom.SetAttributesAsText(nodes[0].NodeID, strings.Join(attrs, " ")).Do(ctx, h)
return dom.SetAttributesAsText(nodes[0].NodeID, strings.Join(attrs, " ")).Do(ctxt, h)
}, opts...)
}
@ -265,7 +265,7 @@ func AttributeValue(sel interface{}, name string, value *string, ok *bool, opts
panic("value cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return errors.New("expected at least one element")
}
@ -295,24 +295,24 @@ func AttributeValue(sel interface{}, name string, value *string, ok *bool, opts
// SetAttributeValue sets the element attribute with name to value for the
// first node matching the selector.
func SetAttributeValue(sel interface{}, name, value string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return dom.SetAttributeValue(nodes[0].NodeID, name, value).Do(ctx, h)
return dom.SetAttributeValue(nodes[0].NodeID, name, value).Do(ctxt, h)
}, opts...)
}
// RemoveAttribute removes the element attribute with name from the first node
// matching the selector.
func RemoveAttribute(sel interface{}, name string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return dom.RemoveAttribute(nodes[0].NodeID, name).Do(ctx, h)
return dom.RemoveAttribute(nodes[0].NodeID, name).Do(ctxt, h)
}, opts...)
}
@ -322,25 +322,25 @@ func JavascriptAttribute(sel interface{}, name string, res interface{}, opts ...
if res == nil {
panic("res cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return EvaluateAsDevTools(fmt.Sprintf(attributeJS, nodes[0].FullXPath(), name), res).Do(ctx, h)
return EvaluateAsDevTools(fmt.Sprintf(attributeJS, nodes[0].FullXPath(), name), res).Do(ctxt, h)
}, opts...)
}
// SetJavascriptAttribute sets the javascript attribute for the first node
// matching the selector.
func SetJavascriptAttribute(sel interface{}, name, value string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var res string
err := EvaluateAsDevTools(fmt.Sprintf(setAttributeJS, nodes[0].FullXPath(), name, value), &res).Do(ctx, h)
err := EvaluateAsDevTools(fmt.Sprintf(setAttributeJS, nodes[0].FullXPath(), name, value), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -370,24 +370,24 @@ func InnerHTML(sel interface{}, html *string, opts ...QueryOption) Action {
// Click sends a mouse click event to the first node matching the selector.
func Click(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return MouseClickNode(nodes[0]).Do(ctx, h)
return MouseClickNode(nodes[0]).Do(ctxt, h)
}, append(opts, NodeVisible)...)
}
// DoubleClick sends a mouse double click event to the first node matching the
// selector.
func DoubleClick(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return MouseClickNode(nodes[0], ClickCount(2)).Do(ctx, h)
return MouseClickNode(nodes[0], ClickCount(2)).Do(ctxt, h)
}, append(opts, NodeVisible)...)
}
@ -397,7 +397,7 @@ func DoubleClick(sel interface{}, opts ...QueryOption) Action {
// Note: when selector matches a input[type="file"] node, then dom.SetFileInputFiles
// is used to set the upload path of the input node to v.
func SendKeys(sel interface{}, v string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
@ -416,22 +416,22 @@ func SendKeys(sel interface{}, v string, opts ...QueryOption) Action {
// when working with input[type="file"], call dom.SetFileInputFiles
if n.NodeName == "INPUT" && typ == "file" {
return dom.SetFileInputFiles([]string{v}).WithNodeID(n.NodeID).Do(ctx, h)
return dom.SetFileInputFiles([]string{v}).WithNodeID(n.NodeID).Do(ctxt, h)
}
return KeyActionNode(n, v).Do(ctx, h)
return KeyActionNode(n, v).Do(ctxt, h)
}, append(opts, NodeVisible)...)
}
// SetUploadFiles sets the files to upload (ie, for a input[type="file"] node)
// for the first node matching the selector.
func SetUploadFiles(sel interface{}, files []string, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
return dom.SetFileInputFiles(files).WithNodeID(nodes[0].NodeID).Do(ctx, h)
return dom.SetFileInputFiles(files).WithNodeID(nodes[0].NodeID).Do(ctxt, h)
}, opts...)
}
@ -441,13 +441,13 @@ func Screenshot(sel interface{}, picbuf *[]byte, opts ...QueryOption) Action {
panic("picbuf cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
// get box model
box, err := dom.GetBoxModel().WithNodeID(nodes[0].NodeID).Do(ctx, h)
box, err := dom.GetBoxModel().WithNodeID(nodes[0].NodeID).Do(ctxt, h)
if err != nil {
return err
}
@ -459,13 +459,13 @@ func Screenshot(sel interface{}, picbuf *[]byte, opts ...QueryOption) Action {
// scroll to node position
var pos []int
err = EvaluateAsDevTools(fmt.Sprintf(scrollJS, int64(box.Margin[0]), int64(box.Margin[1])), &pos).Do(ctx, h)
err = EvaluateAsDevTools(fmt.Sprintf(scrollJS, int64(box.Margin[0]), int64(box.Margin[1])), &pos).Do(ctxt, h)
if err != nil {
return err
}
// take page screenshot
buf, err := page.CaptureScreenshot().Do(ctx, h)
buf, err := page.CaptureScreenshot().Do(ctxt, h)
if err != nil {
return err
}
@ -484,7 +484,8 @@ func Screenshot(sel interface{}, picbuf *[]byte, opts ...QueryOption) Action {
// encode
var croppedBuf bytes.Buffer
if err := png.Encode(&croppedBuf, cropped); err != nil {
err = png.Encode(&croppedBuf, cropped)
if err != nil {
return err
}
@ -497,13 +498,13 @@ func Screenshot(sel interface{}, picbuf *[]byte, opts ...QueryOption) Action {
// Submit is an action that submits the form of the first node matching the
// selector belongs to.
func Submit(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var res bool
err := EvaluateAsDevTools(fmt.Sprintf(submitJS, nodes[0].FullXPath()), &res).Do(ctx, h)
err := EvaluateAsDevTools(fmt.Sprintf(submitJS, nodes[0].FullXPath()), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -519,13 +520,13 @@ func Submit(sel interface{}, opts ...QueryOption) Action {
// Reset is an action that resets the form of the first node matching the
// selector belongs to.
func Reset(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var res bool
err := EvaluateAsDevTools(fmt.Sprintf(resetJS, nodes[0].FullXPath()), &res).Do(ctx, h)
err := EvaluateAsDevTools(fmt.Sprintf(resetJS, nodes[0].FullXPath()), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -544,12 +545,12 @@ func ComputedStyle(sel interface{}, style *[]*css.ComputedProperty, opts ...Quer
panic("style cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
computed, err := css.GetComputedStyleForNode(nodes[0].NodeID).Do(ctx, h)
computed, err := css.GetComputedStyleForNode(nodes[0].NodeID).Do(ctxt, h)
if err != nil {
return err
}
@ -567,7 +568,7 @@ func MatchedStyle(sel interface{}, style **css.GetMatchedStylesForNodeReturns, o
panic("style cannot be nil")
}
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
@ -576,7 +577,7 @@ func MatchedStyle(sel interface{}, style **css.GetMatchedStylesForNodeReturns, o
ret := &css.GetMatchedStylesForNodeReturns{}
ret.InlineStyle, ret.AttributesStyle, ret.MatchedCSSRules,
ret.PseudoElements, ret.Inherited, ret.CSSKeyframesRules,
err = css.GetMatchedStylesForNode(nodes[0].NodeID).Do(ctx, h)
err = css.GetMatchedStylesForNode(nodes[0].NodeID).Do(ctxt, h)
if err != nil {
return err
}
@ -589,13 +590,13 @@ func MatchedStyle(sel interface{}, style **css.GetMatchedStylesForNodeReturns, o
// ScrollIntoView scrolls the window to the first node matching the selector.
func ScrollIntoView(sel interface{}, opts ...QueryOption) Action {
return QueryAfter(sel, func(ctx context.Context, h *Target, nodes ...*cdp.Node) error {
return QueryAfter(sel, func(ctxt context.Context, h *TargetHandler, nodes ...*cdp.Node) error {
if len(nodes) < 1 {
return fmt.Errorf("selector `%s` did not return any nodes", sel)
}
var pos []int
err := EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, nodes[0].FullXPath()), &pos).Do(ctx, h)
err := EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, nodes[0].FullXPath()), &pos).Do(ctxt, h)
if err != nil {
return err
}

527
query_test.go
View File

File diff suppressed because it is too large Load Diff

12
runner/path_darwin.go Normal file
View File

@ -0,0 +1,12 @@
// +build darwin
package runner
const (
// DefaultChromePath is the default path to the Chrome application.
DefaultChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`
)
func findChromePath() string {
return DefaultChromePath
}

31
runner/path_unix.go Normal file
View File

@ -0,0 +1,31 @@
// +build linux freebsd netbsd openbsd
package runner
import "os/exec"
const (
// DefaultChromePath is the default path to the google-chrome executable if
// a variant cannot be found on $PATH.
DefaultChromePath = "/usr/bin/google-chrome"
)
// chromeNames are the Chrome executable names to search for in the path.
var chromeNames = []string{
"google-chrome",
"chromium-browser",
"chromium",
"google-chrome-beta",
"google-chrome-unstable",
}
func findChromePath() string {
for _, p := range chromeNames {
path, err := exec.LookPath(p)
if err == nil {
return path
}
}
return DefaultChromePath
}

33
runner/path_windows.go Normal file
View File

@ -0,0 +1,33 @@
// +build windows
package runner
import "os/exec"
const (
// DefaultChromePath is the default path to use for Google Chrome if the
// executable is not in %PATH%.
DefaultChromePath = `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`
// DefaultEdgeDiagnosticsAdapterPath is the default path to use for the
// Microsoft Edge Diagnostics Adapter if the executable is not in %PATH%.
DefaultEdgeDiagnosticsAdapterPath = `c:\Edge\EdgeDiagnosticsAdapter\x64\EdgeDiagnosticsAdapter.exe`
)
func findChromePath() string {
path, err := exec.LookPath(`chrome.exe`)
if err == nil {
return path
}
return DefaultChromePath
}
func findEdgePath() string {
path, err := exec.LookPath(`EdgeDiagnosticsAdapter.exe`)
if err == nil {
return path
}
return DefaultEdgeDiagnosticsAdapterPath
}

457
runner/runner.go Normal file
View File

@ -0,0 +1,457 @@
// Package runner provides a Chrome process runner.
package runner
import (
"context"
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"os/exec"
"regexp"
"runtime"
"sync"
"syscall"
"github.com/chromedp/chromedp/client"
)
const (
// DefaultUserDataDirPrefix is the default user data directory prefix.
DefaultUserDataDirPrefix = "chromedp-runner.%d."
)
// Runner holds information about a running Chrome process.
type Runner struct {
opts map[string]interface{}
cmd *exec.Cmd
waiting bool
rw sync.RWMutex
}
// New creates a new Chrome process using the supplied command line options.
func New(opts ...CommandLineOption) (*Runner, error) {
var err error
cliOpts := map[string]interface{}{}
// apply opts
for _, o := range opts {
err = o(cliOpts)
if err != nil {
return nil, err
}
}
// set default Chrome options if exec-path not provided
if _, ok := cliOpts["exec-path"]; !ok {
cliOpts["exec-path"] = findChromePath()
for k, v := range map[string]interface{}{
"no-first-run": true,
"no-default-browser-check": true,
"remote-debugging-port": 9222,
} {
if _, ok := cliOpts[k]; !ok {
cliOpts[k] = v
}
}
}
// add KillProcessGroup and ForceKill if no other cmd opts provided
if _, ok := cliOpts["cmd-opts"]; !ok {
for _, o := range []CommandLineOption{KillProcessGroup, ForceKill} {
err = o(cliOpts)
if err != nil {
return nil, err
}
}
}
return &Runner{
opts: cliOpts,
}, nil
}
// cliOptRE is a regular expression to validate a chrome cli option.
var cliOptRE = regexp.MustCompile(`^[a-z0-9\-]+$`)
// buildOpts generates the command line options for Chrome.
func (r *Runner) buildOpts() []string {
var opts []string
// process options
var urlstr string
for k, v := range r.opts {
if !cliOptRE.MatchString(k) || v == nil {
continue
}
switch k {
case "exec-path", "cmd-opts", "process-opts":
continue
case "start-url":
urlstr = v.(string)
default:
switch z := v.(type) {
case bool:
if z {
opts = append(opts, "--"+k)
}
case string:
opts = append(opts, "--"+k+"="+z)
default:
opts = append(opts, "--"+k+"="+fmt.Sprintf("%v", v))
}
}
}
if urlstr == "" {
urlstr = "about:blank"
}
return append(opts, urlstr)
}
// Start starts a Chrome process using the specified context. The Chrome
// process can be terminated by closing the passed context.
func (r *Runner) Start(ctxt context.Context) error {
var err error
var ok bool
r.rw.RLock()
cmd := r.cmd
r.rw.RUnlock()
if cmd != nil {
return errors.New("already started")
}
// setup context
if ctxt == nil {
ctxt = context.Background()
}
// set user data dir, if not provided
_, ok = r.opts["user-data-dir"]
if !ok {
r.opts["user-data-dir"], err = ioutil.TempDir(
defaultUserDataTmpDir, fmt.Sprintf(DefaultUserDataDirPrefix, r.Port()),
)
if err != nil {
return err
}
}
// ensure exec-path set
execPath, ok := r.opts["exec-path"]
if !ok {
return errors.New("exec-path command line option not set, or chrome executable not found in $PATH")
}
// create cmd
r.cmd = exec.CommandContext(ctxt, execPath.(string), r.buildOpts()...)
// apply cmd opts
if cmdOpts, ok := r.opts["cmd-opts"]; ok {
for _, co := range cmdOpts.([]func(*exec.Cmd) error) {
err = co(r.cmd)
if err != nil {
return err
}
}
}
// start process
err = r.cmd.Start()
if err != nil {
return err
}
// apply process opts
if processOpts, ok := r.opts["process-opts"]; ok {
for _, po := range processOpts.([]func(*os.Process) error) {
err = po(r.cmd.Process)
if err != nil {
// TODO: do something better here, as we want to kill
// the child process, do cleanup, etc.
panic(err)
//return err
}
}
}
return nil
}
// Shutdown shuts down the Chrome process.
func (r *Runner) Shutdown(ctxt context.Context, opts ...client.Option) error {
var err error
cl := r.Client(opts...)
targets, err := cl.ListPageTargets(ctxt)
if err != nil {
return err
}
var wg sync.WaitGroup
errs := make([]error, len(targets))
for i, t := range targets {
wg.Add(1)
go func(wg *sync.WaitGroup, i int, t client.Target) {
defer wg.Done()
errs[i] = cl.CloseTarget(ctxt, t)
}(&wg, i, t)
}
wg.Wait()
for _, e := range errs {
if e != nil {
return e
}
}
// osx applications do not automatically exit when all windows (ie, tabs)
// closed, so send SIGTERM.
//
// TODO: add other behavior here for more process options on shutdown?
if runtime.GOOS == "darwin" && r.cmd != nil && r.cmd.Process != nil {
return r.cmd.Process.Signal(syscall.SIGTERM)
}
return nil
}
// Wait waits for the previously started Chrome process to terminate, returning
// any encountered error.
func (r *Runner) Wait() error {
r.rw.RLock()
waiting := r.waiting
r.rw.RUnlock()
if waiting {
return errors.New("already waiting")
}
r.rw.Lock()
r.waiting = true
r.rw.Unlock()
defer func() {
r.rw.Lock()
r.waiting = false
r.rw.Unlock()
}()
return r.cmd.Wait()
}
// Port returns the port the process was launched with.
func (r *Runner) Port() int {
var port interface{}
var ok bool
port, ok = r.opts["remote-debugging-port"]
if !ok {
port, ok = r.opts["port"]
}
if !ok {
panic("expected either remote-debugging-port or port to be specified in command line options")
}
var p int
p, ok = port.(int)
if !ok {
panic("expected port to be type int")
}
return p
}
// Client returns a Chrome Debugging Protocol client for the running Chrome
// process.
func (r *Runner) Client(opts ...client.Option) *client.Client {
return client.New(append(opts,
client.URL(fmt.Sprintf("http://localhost:%d/json", r.Port())),
)...)
}
// WatchPageTargets returns a channel that will receive new page targets as
// they are created.
func (r *Runner) WatchPageTargets(ctxt context.Context, opts ...client.Option) <-chan client.Target {
return r.Client(opts...).WatchPageTargets(ctxt)
}
// Run starts a new Chrome process runner, using the provided context and
// command line options.
func Run(ctxt context.Context, opts ...CommandLineOption) (*Runner, error) {
var err error
// create
r, err := New(opts...)
if err != nil {
return nil, err
}
// start
err = r.Start(ctxt)
if err != nil {
return nil, err
}
return r, nil
}
// CommandLineOption is a Chrome command line option.
//
// see: http://peter.sh/experiments/chromium-command-line-switches/
type CommandLineOption func(map[string]interface{}) error
// Flag is a generic Chrome command line option to pass a name=value flag to
// Chrome.
func Flag(name string, value interface{}) CommandLineOption {
return func(m map[string]interface{}) error {
m[name] = value
return nil
}
}
// Path sets the path to the Chrome executable and sets default run options for
// Chrome. This will also set the remote debugging port to 9222, and disable
// the first run / default browser check.
//
// Note: use ExecPath if you do not want to set other options.
func Path(path string) CommandLineOption {
return func(m map[string]interface{}) error {
m["exec-path"] = path
m["no-first-run"] = true
m["no-default-browser-check"] = true
m["remote-debugging-port"] = 9222
return nil
}
}
// HeadlessPathPort is the Chrome command line option to set the default
// settings for running the headless_shell executable. If path is empty, then
// an attempt will be made to find headless_shell on the path.
func HeadlessPathPort(path string, port int) CommandLineOption {
if path == "" {
path, _ = exec.LookPath("headless_shell")
}
return func(m map[string]interface{}) error {
m["exec-path"] = path
m["remote-debugging-port"] = port
m["headless"] = true
return nil
}
}
// ExecPath is a Chrome command line option to set the exec path.
func ExecPath(path string) CommandLineOption {
return Flag("exec-path", path)
}
// Port is the Chrome command line option to set the remote debugging port.
func Port(port int) CommandLineOption {
return Flag("remote-debugging-port", port)
}
// UserDataDir is the Chrome command line option to set the user data dir.
//
// Note: set this option to manually set the profile directory used by Chrome.
// When this is not set, then a default path will be created in the /tmp
// directory.
func UserDataDir(dir string) CommandLineOption {
return Flag("user-data-dir", dir)
}
// StartURL is the Chrome command line option to set the initial URL.
func StartURL(urlstr string) CommandLineOption {
return Flag("start-url", urlstr)
}
// Proxy is the Chrome command line option to set the outbound proxy.
func Proxy(proxy string) CommandLineOption {
return Flag("proxy-server", proxy)
}
// ProxyPacURL is the Chrome command line option to set the URL of a proxy PAC file.
func ProxyPacURL(pacURL url.URL) CommandLineOption {
return Flag("proxy-pac-url", pacURL.String())
}
// WindowSize is the Chrome command line option to set the initial window size.
func WindowSize(width, height int) CommandLineOption {
return Flag("window-size", fmt.Sprintf("%d,%d", width, height))
}
// UserAgent is the Chrome command line option to set the default User-Agent
// header.
func UserAgent(userAgent string) CommandLineOption {
return Flag("user-agent", userAgent)
}
// NoSandbox is the Chrome comamnd line option to disable the sandbox.
func NoSandbox(m map[string]interface{}) error {
return Flag("no-sandbox", true)(m)
}
// NoFirstRun is the Chrome comamnd line option to disable the first run
// dialog.
func NoFirstRun(m map[string]interface{}) error {
return Flag("no-first-run", true)(m)
}
// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the
// default browser check.
func NoDefaultBrowserCheck(m map[string]interface{}) error {
return Flag("no-default-browser-check", true)(m)
}
// DisableGPU is the Chrome command line option to disable the GPU process.
func DisableGPU(m map[string]interface{}) error {
return Flag("disable-gpu", true)(m)
}
// CmdOpt is a Chrome command line option to modify the underlying exec.Cmd
// prior to invocation.
func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {
return func(m map[string]interface{}) error {
var opts []func(*exec.Cmd) error
if e, ok := m["cmd-opts"]; ok {
opts, ok = e.([]func(*exec.Cmd) error)
if !ok {
return errors.New("cmd-opts is in invalid state")
}
}
m["cmd-opts"] = append(opts, o)
return nil
}
}
// ProcessOpt is a Chrome command line option to modify the child os.Process
// after started exec.Cmd.Start.
func ProcessOpt(o func(*os.Process) error) CommandLineOption {
return func(m map[string]interface{}) error {
var opts []func(*os.Process) error
if e, ok := m["process-opts"]; ok {
opts, ok = e.([]func(*os.Process) error)
if !ok {
return errors.New("process-opts is in invalid state")
}
}
m["process-opts"] = append(opts, o)
return nil
}
}

11
runner/runner_bsd.go Normal file
View File

@ -0,0 +1,11 @@
// +build darwin freebsd netbsd openbsd
package runner
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return nil
}

87
runner/runner_linux.go Normal file
View File

@ -0,0 +1,87 @@
// +build linux
package runner
import (
"os"
"os/exec"
"syscall"
"unsafe"
)
// ByteCount is a type byte count const.
type ByteCount uint64
// ByteCount values.
const (
Byte ByteCount = 1
Kilobyte ByteCount = 1024 * Byte
Megabyte ByteCount = 1024 * Kilobyte
Gigabyte ByteCount = 1024 * Megabyte
)
// prlimit invokes the system's prlimit call. Copied from Go source tree.
//
// Note: this needs either the CAP_SYS_RESOURCE capability, or the invoking
// process needs to have the same functional user and group as the pid being
// modified.
//
// see: man 2 prlimit
func prlimit(pid int, res int, newv, old *syscall.Rlimit) error {
_, _, err := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, uintptr(pid), uintptr(res), uintptr(unsafe.Pointer(newv)), uintptr(unsafe.Pointer(old)), 0, 0)
if err != 0 {
return err
}
return nil
}
// Rlimit is a Chrome command line option to set the soft rlimit value for res
// on a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child process
// has been started.
//
// see: man 2 prlimit
func Rlimit(res int, cur, max uint64) CommandLineOption {
return ProcessOpt(func(p *os.Process) error {
return prlimit(p.Pid, syscall.RLIMIT_AS, &syscall.Rlimit{
Cur: cur,
Max: max,
}, nil)
})
}
// LimitMemory is a Chrome command line option to set the soft memory limit for
// a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child
// process has been started.
func LimitMemory(mem ByteCount) CommandLineOption {
return Rlimit(syscall.RLIMIT_AS, uint64(mem), uint64(mem))
}
// LimitCoreDump is a Chrome command line option to set the soft core dump
// limit for a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child
// process has been started.
func LimitCoreDump(sz ByteCount) CommandLineOption {
return Rlimit(syscall.RLIMIT_CORE, uint64(sz), uint64(sz))
}
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return CmdOpt(func(c *exec.Cmd) error {
if c.SysProcAttr == nil {
c.SysProcAttr = new(syscall.SysProcAttr)
}
c.SysProcAttr.Pdeathsig = syscall.SIGKILL
return nil
})(m)
}

31
runner/runner_unix.go Normal file
View File

@ -0,0 +1,31 @@
// +build linux darwin freebsd netbsd openbsd
package runner
import (
"os/exec"
"syscall"
)
var (
// DefaultUserDataTmpDir is the default directory path for created user
// data directories.
defaultUserDataTmpDir = "/tmp"
)
// KillProcessGroup is a Chrome command line option that will instruct the
// invoked child Chrome process to terminate when the parent process (ie, the
// Go application) dies.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true and does nothing on Windows.
func KillProcessGroup(m map[string]interface{}) error {
return CmdOpt(func(c *exec.Cmd) error {
if c.SysProcAttr == nil {
c.SysProcAttr = new(syscall.SysProcAttr)
}
c.SysProcAttr.Setpgid = true
return nil
})(m)
}

44
runner/runner_windows.go Normal file
View File

@ -0,0 +1,44 @@
// +build windows
package runner
import "os"
var (
defaultUserDataTmpDir = os.Getenv("USERPROFILE") + `\AppData\Local`
)
// KillProcessGroup is a Chrome command line option that will instruct the
// invoked child Chrome process to terminate when the parent process (ie, the
// Go application) dies.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true and does nothing on Windows.
func KillProcessGroup(m map[string]interface{}) error {
return nil
}
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return nil
}
// EdgeDiagnosticsAdapterWithPath is a command line option to specify using the
// Microsoft Edge Diagnostics adapter at the specified path.
func EdgeDiagnosticsAdapterWithPathAndPort(path string, port int) CommandLineOption {
return func(m map[string]interface{}) error {
m["exec-path"] = path
m["port"] = port
return nil
}
}
// EdgeDiagnosticsAdapter is a command line option to specify using the
// Microsoft Edge Diagnostics adapter found on the path.
//
// If the
func EdgeDiagnosticsAdapter() CommandLineOption {
return EdgeDiagnosticsAdapterWithPathAndPort(findEdgePath(), 9222)
}

172
sel.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"strings"
"sync"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/dom"
@ -25,9 +26,9 @@ tagname
type Selector struct {
sel interface{}
exp int
by func(context.Context, *Target, *cdp.Node) ([]cdp.NodeID, error)
wait func(context.Context, *Target, *cdp.Frame, ...cdp.NodeID) ([]*cdp.Node, error)
after func(context.Context, *Target, ...*cdp.Node) error
by func(context.Context, *TargetHandler, *cdp.Node) ([]cdp.NodeID, error)
wait func(context.Context, *TargetHandler, *cdp.Node, ...cdp.NodeID) ([]*cdp.Node, error)
after func(context.Context, *TargetHandler, ...*cdp.Node) error
}
// Query is an action to query for document nodes match the specified sel and
@ -55,17 +56,21 @@ func Query(sel interface{}, opts ...QueryOption) Action {
}
// Do satisfies the Action interface.
func (s *Selector) Do(ctx context.Context, h cdp.Executor) error {
th, ok := h.(*Target)
func (s *Selector) Do(ctxt context.Context, h cdp.Executor) error {
th, ok := h.(*TargetHandler)
if !ok {
return ErrInvalidHandler
}
// TODO: fix this
ctxt, cancel := context.WithTimeout(ctxt, 100*time.Second)
defer cancel()
var err error
select {
case err = <-s.run(ctx, th):
case <-ctx.Done():
err = ctx.Err()
case err = <-s.run(ctxt, th):
case <-ctxt.Done():
err = ctxt.Err()
}
return err
@ -74,35 +79,54 @@ func (s *Selector) Do(ctx context.Context, h cdp.Executor) error {
// run runs the selector action, starting over if the original returned nodes
// are invalidated prior to finishing the selector's by, wait, check, and after
// funcs.
func (s *Selector) run(ctx context.Context, h *Target) chan error {
ch := make(chan error, 1)
h.waitQueue <- func(cur *cdp.Frame) bool {
cur.RLock()
root := cur.Root
cur.RUnlock()
func (s *Selector) run(ctxt context.Context, h *TargetHandler) chan error {
ch := make(chan error)
if root == nil {
// not ready?
return false
go func() {
defer close(ch)
for {
root, err := h.GetRoot(ctxt)
if err != nil {
select {
case <-ctxt.Done():
ch <- ctxt.Err()
return
default:
continue
}
}
ids, err := s.by(ctx, h, root)
if err != nil || len(ids) < s.exp {
return false
select {
default:
ids, err := s.by(ctxt, h, root)
if err == nil && len(ids) >= s.exp {
nodes, err := s.wait(ctxt, h, root, ids...)
if err == nil {
if s.after == nil {
return
}
nodes, err := s.wait(ctx, h, cur, ids...)
// if nodes==nil, we're not yet ready
if nodes == nil || err != nil {
return false
}
if s.after != nil {
if err := s.after(ctx, h, nodes...); err != nil {
err = s.after(ctxt, h, nodes...)
if err != nil {
ch <- err
}
return
}
close(ch)
return true
}
time.Sleep(DefaultCheckDuration)
case <-root.Invalidated:
continue
case <-ctxt.Done():
ch <- ctxt.Err()
return
}
}
}()
return ch
}
@ -115,10 +139,20 @@ func (s *Selector) selAsString() string {
return fmt.Sprintf("%s", s.sel)
}
// selAsInt forces sel into a int.
/*func (s *Selector) selAsInt() int {
sel, ok := s.sel.(int)
if !ok {
panic("selector must be int")
}
return sel
}*/
// QueryAfter is an action that will match the specified sel using the supplied
// query options, and after the visibility conditions of the query have been
// met, will execute f.
func QueryAfter(sel interface{}, f func(context.Context, *Target, ...*cdp.Node) error, opts ...QueryOption) Action {
func QueryAfter(sel interface{}, f func(context.Context, *TargetHandler, ...*cdp.Node) error, opts ...QueryOption) Action {
return Query(sel, append(opts, After(f))...)
}
@ -126,7 +160,7 @@ func QueryAfter(sel interface{}, f func(context.Context, *Target, ...*cdp.Node)
type QueryOption func(*Selector)
// ByFunc is a query option to set the func used to select elements.
func ByFunc(f func(context.Context, *Target, *cdp.Node) ([]cdp.NodeID, error)) QueryOption {
func ByFunc(f func(context.Context, *TargetHandler, *cdp.Node) ([]cdp.NodeID, error)) QueryOption {
return func(s *Selector) {
s.by = f
}
@ -135,8 +169,8 @@ func ByFunc(f func(context.Context, *Target, *cdp.Node) ([]cdp.NodeID, error)) Q
// ByQuery is a query option to select a single element using
// DOM.querySelector.
func ByQuery(s *Selector) {
ByFunc(func(ctx context.Context, h *Target, n *cdp.Node) ([]cdp.NodeID, error) {
nodeID, err := dom.QuerySelector(n.NodeID, s.selAsString()).Do(ctx, h)
ByFunc(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) ([]cdp.NodeID, error) {
nodeID, err := dom.QuerySelector(n.NodeID, s.selAsString()).Do(ctxt, h)
if err != nil {
return nil, err
}
@ -151,8 +185,8 @@ func ByQuery(s *Selector) {
// ByQueryAll is a query option to select elements by DOM.querySelectorAll.
func ByQueryAll(s *Selector) {
ByFunc(func(ctx context.Context, h *Target, n *cdp.Node) ([]cdp.NodeID, error) {
return dom.QuerySelectorAll(n.NodeID, s.selAsString()).Do(ctx, h)
ByFunc(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) ([]cdp.NodeID, error) {
return dom.QuerySelectorAll(n.NodeID, s.selAsString()).Do(ctxt, h)
})(s)
}
@ -165,8 +199,8 @@ func ByID(s *Selector) {
// BySearch is a query option via DOM.performSearch (works with both CSS and
// XPath queries).
func BySearch(s *Selector) {
ByFunc(func(ctx context.Context, h *Target, n *cdp.Node) ([]cdp.NodeID, error) {
id, count, err := dom.PerformSearch(s.selAsString()).Do(ctx, h)
ByFunc(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) ([]cdp.NodeID, error) {
id, count, err := dom.PerformSearch(s.selAsString()).Do(ctxt, h)
if err != nil {
return nil, err
}
@ -175,7 +209,7 @@ func BySearch(s *Selector) {
return []cdp.NodeID{}, nil
}
nodes, err := dom.GetSearchResults(id, 0, count).Do(ctx, h)
nodes, err := dom.GetSearchResults(id, 0, count).Do(ctxt, h)
if err != nil {
return nil, err
}
@ -191,9 +225,9 @@ func ByNodeID(s *Selector) {
panic("ByNodeID can only work on []cdp.NodeID")
}
ByFunc(func(ctx context.Context, h *Target, n *cdp.Node) ([]cdp.NodeID, error) {
ByFunc(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) ([]cdp.NodeID, error) {
for _, id := range ids {
err := dom.RequestChildNodes(id).WithPierce(true).Do(ctx, h)
err := dom.RequestChildNodes(id).WithPierce(true).Do(ctxt, h)
if err != nil {
return nil, err
}
@ -204,28 +238,38 @@ func ByNodeID(s *Selector) {
}
// waitReady waits for the specified nodes to be ready.
func (s *Selector) waitReady(check func(context.Context, *Target, *cdp.Node) error) func(context.Context, *Target, *cdp.Frame, ...cdp.NodeID) ([]*cdp.Node, error) {
return func(ctx context.Context, h *Target, cur *cdp.Frame, ids ...cdp.NodeID) ([]*cdp.Node, error) {
func (s *Selector) waitReady(check func(context.Context, *TargetHandler, *cdp.Node) error) func(context.Context, *TargetHandler, *cdp.Node, ...cdp.NodeID) ([]*cdp.Node, error) {
return func(ctxt context.Context, h *TargetHandler, n *cdp.Node, ids ...cdp.NodeID) ([]*cdp.Node, error) {
f, err := h.WaitFrame(ctxt, cdp.EmptyFrameID)
if err != nil {
return nil, err
}
wg := new(sync.WaitGroup)
nodes := make([]*cdp.Node, len(ids))
cur.RLock()
errs := make([]error, len(ids))
for i, id := range ids {
nodes[i] = cur.Nodes[id]
if nodes[i] == nil {
cur.RUnlock()
// not yet ready
return nil, nil
wg.Add(1)
go func(i int, id cdp.NodeID) {
defer wg.Done()
nodes[i], errs[i] = h.WaitNode(ctxt, f, id)
}(i, id)
}
wg.Wait()
for _, err := range errs {
if err != nil {
return nil, err
}
}
cur.RUnlock()
if check != nil {
var wg sync.WaitGroup
errs := make([]error, len(nodes))
for i, n := range nodes {
wg.Add(1)
go func(i int, n *cdp.Node) {
defer wg.Done()
errs[i] = check(ctx, h, n)
errs[i] = check(ctxt, h, n)
}(i, n)
}
wg.Wait()
@ -242,7 +286,7 @@ func (s *Selector) waitReady(check func(context.Context, *Target, *cdp.Node) err
}
// WaitFunc is a query option to set a custom wait func.
func WaitFunc(wait func(context.Context, *Target, *cdp.Frame, ...cdp.NodeID) ([]*cdp.Node, error)) QueryOption {
func WaitFunc(wait func(context.Context, *TargetHandler, *cdp.Node, ...cdp.NodeID) ([]*cdp.Node, error)) QueryOption {
return func(s *Selector) {
s.wait = wait
}
@ -255,9 +299,9 @@ func NodeReady(s *Selector) {
// NodeVisible is a query option to wait until the element is visible.
func NodeVisible(s *Selector) {
WaitFunc(s.waitReady(func(ctx context.Context, h *Target, n *cdp.Node) error {
WaitFunc(s.waitReady(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) error {
// check box model
_, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctx, h)
_, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctxt, h)
if err != nil {
if isCouldNotComputeBoxModelError(err) {
return ErrNotVisible
@ -268,7 +312,7 @@ func NodeVisible(s *Selector) {
// check offsetParent
var res bool
err = EvaluateAsDevTools(fmt.Sprintf(visibleJS, n.FullXPath()), &res).Do(ctx, h)
err = EvaluateAsDevTools(fmt.Sprintf(visibleJS, n.FullXPath()), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -281,9 +325,9 @@ func NodeVisible(s *Selector) {
// NodeNotVisible is a query option to wait until the element is not visible.
func NodeNotVisible(s *Selector) {
WaitFunc(s.waitReady(func(ctx context.Context, h *Target, n *cdp.Node) error {
WaitFunc(s.waitReady(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) error {
// check box model
_, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctx, h)
_, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(ctxt, h)
if err != nil {
if isCouldNotComputeBoxModelError(err) {
return nil
@ -294,7 +338,7 @@ func NodeNotVisible(s *Selector) {
// check offsetParent
var res bool
err = EvaluateAsDevTools(fmt.Sprintf(visibleJS, n.FullXPath()), &res).Do(ctx, h)
err = EvaluateAsDevTools(fmt.Sprintf(visibleJS, n.FullXPath()), &res).Do(ctxt, h)
if err != nil {
return err
}
@ -307,7 +351,7 @@ func NodeNotVisible(s *Selector) {
// NodeEnabled is a query option to wait until the element is enabled.
func NodeEnabled(s *Selector) {
WaitFunc(s.waitReady(func(ctx context.Context, h *Target, n *cdp.Node) error {
WaitFunc(s.waitReady(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) error {
n.RLock()
defer n.RUnlock()
@ -323,7 +367,7 @@ func NodeEnabled(s *Selector) {
// NodeSelected is a query option to wait until the element is selected.
func NodeSelected(s *Selector) {
WaitFunc(s.waitReady(func(ctx context.Context, h *Target, n *cdp.Node) error {
WaitFunc(s.waitReady(func(ctxt context.Context, h *TargetHandler, n *cdp.Node) error {
n.RLock()
defer n.RUnlock()
@ -337,11 +381,11 @@ func NodeSelected(s *Selector) {
}))(s)
}
// NodeNotPresent is a query option to wait until no elements are present
// matching the selector.
// NodeNotPresent is a query option to wait until no elements match are
// present matching the selector.
func NodeNotPresent(s *Selector) {
s.exp = 0
WaitFunc(func(ctx context.Context, h *Target, cur *cdp.Frame, ids ...cdp.NodeID) ([]*cdp.Node, error) {
WaitFunc(func(ctxt context.Context, h *TargetHandler, n *cdp.Node, ids ...cdp.NodeID) ([]*cdp.Node, error) {
if len(ids) != 0 {
return nil, ErrHasResults
}
@ -359,7 +403,7 @@ func AtLeast(n int) QueryOption {
// After is a query option to set a func that will be executed after the wait
// has succeeded.
func After(f func(context.Context, *Target, ...*cdp.Node) error) QueryOption {
func After(f func(context.Context, *TargetHandler, ...*cdp.Node) error) QueryOption {
return func(s *Selector) {
s.after = f
}

163
sel_test.go
View File

@ -9,21 +9,26 @@ import (
func TestWaitReady(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
var nodeIDs []cdp.NodeID
if err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID)); err != nil {
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, WaitReady("#input2", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
if err := Run(ctx,
WaitReady("#input2", ByID),
Value(nodeIDs, &value, ByNodeID),
); err != nil {
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
@ -31,21 +36,26 @@ func TestWaitReady(t *testing.T) {
func TestWaitVisible(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
var nodeIDs []cdp.NodeID
if err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID)); err != nil {
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, WaitVisible("#input2", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
if err := Run(ctx,
WaitVisible("#input2", ByID),
Value(nodeIDs, &value, ByNodeID),
); err != nil {
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
@ -53,22 +63,31 @@ func TestWaitVisible(t *testing.T) {
func TestWaitNotVisible(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
var nodeIDs []cdp.NodeID
if err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID)); err != nil {
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, Click("#button2", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitNotVisible("#input2", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
if err := Run(ctx,
Click("#button2", ByID),
WaitNotVisible("#input2", ByID),
Value(nodeIDs, &value, ByNodeID),
); err != nil {
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
@ -76,35 +95,46 @@ func TestWaitNotVisible(t *testing.T) {
func TestWaitEnabled(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
var attr string
var ok bool
if err := Run(ctx, AttributeValue("#select1", "disabled", &attr, &ok, ByID)); err != nil {
err := c.Run(defaultContext, AttributeValue("#select1", "disabled", &attr, &ok, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if !ok {
t.Fatal("expected element to be disabled")
}
if err := Run(ctx,
Click("#button3", ByID),
WaitEnabled("#select1", ByID),
AttributeValue("#select1", "disabled", &attr, &ok, ByID),
); err != nil {
err = c.Run(defaultContext, Click("#button3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitEnabled("#select1", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, AttributeValue("#select1", "disabled", &attr, &ok, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected element to be enabled")
}
var value string
if err := Run(ctx,
SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"),
Value("#select1", &value, ByID),
); err != nil {
err = c.Run(defaultContext, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value("#select1", &value, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if value != "foo" {
t.Fatalf("expected value to be foo, got: %s", value)
}
@ -113,32 +143,43 @@ func TestWaitEnabled(t *testing.T) {
func TestWaitSelected(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
if err := Run(ctx,
Click("#button3", ByID),
WaitEnabled("#select1", ByID),
); err != nil {
err := c.Run(defaultContext, Click("#button3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitEnabled("#select1", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
var attr string
ok := false
if err := Run(ctx, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, &ok)); err != nil {
err = c.Run(defaultContext, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, &ok))
if err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected element to be not selected")
}
if err := Run(ctx,
SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"),
WaitSelected(`//*[@id="select1"]/option[1]`),
AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, nil),
); err != nil {
err = c.Run(defaultContext, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitSelected(`//*[@id="select1"]/option[1]`))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, nil))
if err != nil {
t.Fatalf("got error: %v", err)
}
if attr != "true" {
t.Fatal("expected element to be selected")
}
@ -147,14 +188,21 @@ func TestWaitSelected(t *testing.T) {
func TestWaitNotPresent(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
if err := Run(ctx,
WaitVisible("#input3", ByID),
Click("#button4", ByID),
WaitNotPresent("#input3", ByID),
); err != nil {
err := c.Run(defaultContext, WaitVisible("#input3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, Click("#button4", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitNotPresent("#input3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
@ -162,11 +210,12 @@ func TestWaitNotPresent(t *testing.T) {
func TestAtLeast(t *testing.T) {
t.Parallel()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
c := testAllocate(t, "js.html")
defer c.Release()
var nodes []*cdp.Node
if err := Run(ctx, Nodes("//input", &nodes, AtLeast(3))); err != nil {
err := c.Run(defaultContext, Nodes("//input", &nodes, AtLeast(3)))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) < 3 {

320
target.go
View File

@ -1,320 +0,0 @@
package chromedp
import (
"context"
"encoding/json"
"strings"
"sync/atomic"
"time"
"github.com/mailru/easyjson"
"github.com/chromedp/cdproto"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/inspector"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/target"
)
// Target manages a Chrome DevTools Protocol target.
type Target struct {
browser *Browser
SessionID target.SessionID
TargetID target.ID
waitQueue chan func(cur *cdp.Frame) bool
eventQueue chan *cdproto.Message
// below are the old TargetHandler fields.
// frames is the set of encountered frames.
frames map[cdp.FrameID]*cdp.Frame
// cur is the current top level frame.
cur *cdp.Frame
// logging funcs
logf, errf func(string, ...interface{})
}
func (t *Target) run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case msg := <-t.eventQueue:
if err := t.processEvent(ctx, msg); err != nil {
t.errf("could not process event: %v", err)
continue
}
default:
// prevent busy spinning. TODO: do better
time.Sleep(5 * time.Millisecond)
n := len(t.waitQueue)
if n == 0 {
continue
}
if t.cur == nil {
continue
}
for i := 0; i < n; i++ {
fn := <-t.waitQueue
if !fn(t.cur) {
// try again later.
t.waitQueue <- fn
}
}
}
}
}
func (t *Target) Execute(ctx context.Context, method string, params json.Marshaler, res json.Unmarshaler) error {
paramsMsg := emptyObj
if params != nil {
var err error
if paramsMsg, err = json.Marshal(params); err != nil {
return err
}
}
innerID := atomic.AddInt64(&t.browser.next, 1)
msg := &cdproto.Message{
ID: innerID,
Method: cdproto.MethodType(method),
Params: paramsMsg,
}
msgJSON, err := json.Marshal(msg)
if err != nil {
return err
}
sendParams := target.SendMessageToTarget(string(msgJSON)).
WithSessionID(t.SessionID)
sendParamsJSON, _ := json.Marshal(sendParams)
// We want to grab the response from the inner message.
ch := make(chan *cdproto.Message, 1)
t.browser.cmdQueue <- cmdJob{
msg: &cdproto.Message{ID: innerID},
resp: ch,
}
// The response from the outer message is uninteresting; pass a nil
// resp channel.
outerID := atomic.AddInt64(&t.browser.next, 1)
t.browser.cmdQueue <- cmdJob{
msg: &cdproto.Message{
ID: outerID,
Method: target.CommandSendMessageToTarget,
Params: sendParamsJSON,
},
}
select {
case msg := <-ch:
switch {
case msg == nil:
return ErrChannelClosed
case msg.Error != nil:
return msg.Error
case res != nil:
return json.Unmarshal(msg.Result, res)
}
case <-ctx.Done():
return ctx.Err()
}
return nil
}
// below are the old TargetHandler methods.
// processEvent processes an incoming event.
func (t *Target) processEvent(ctx context.Context, msg *cdproto.Message) error {
if msg == nil {
return ErrChannelClosed
}
// unmarshal
ev, err := cdproto.UnmarshalMessage(msg)
if err != nil {
if strings.Contains(err.Error(), "unknown command or event") {
// This is most likely an event received from an older
// Chrome which a newer cdproto doesn't have, as it is
// deprecated. Ignore that error.
// TODO: use error wrapping once Go 1.13 is released.
return nil
}
return err
}
switch ev.(type) {
case *inspector.EventDetached:
return nil
case *dom.EventDocumentUpdated:
t.documentUpdated(ctx)
return nil
}
switch msg.Method.Domain() {
case "Page":
t.pageEvent(ev)
case "DOM":
t.domEvent(ev)
}
return nil
}
// documentUpdated handles the document updated event, retrieving the document
// root for the root frame.
func (t *Target) documentUpdated(ctx context.Context) {
f := t.cur
f.Lock()
defer f.Unlock()
// invalidate nodes
if f.Root != nil {
close(f.Root.Invalidated)
}
f.Nodes = make(map[cdp.NodeID]*cdp.Node)
var err error
f.Root, err = dom.GetDocument().WithPierce(true).Do(ctx, t)
if err == context.Canceled {
return // TODO: perhaps not necessary, but useful to keep the tests less noisy
}
if err != nil {
t.errf("could not retrieve document root for %s: %v", f.ID, err)
return
}
f.Root.Invalidated = make(chan struct{})
walk(f.Nodes, f.Root)
}
// emptyObj is an empty JSON object message.
var emptyObj = easyjson.RawMessage([]byte(`{}`))
// pageEvent handles incoming page events.
func (t *Target) pageEvent(ev interface{}) {
var id cdp.FrameID
var op frameOp
switch e := ev.(type) {
case *page.EventFrameNavigated:
t.frames[e.Frame.ID] = e.Frame
t.cur = e.Frame
return
case *page.EventFrameAttached:
id, op = e.FrameID, frameAttached(e.ParentFrameID)
case *page.EventFrameDetached:
id, op = e.FrameID, frameDetached
case *page.EventFrameStartedLoading:
id, op = e.FrameID, frameStartedLoading
case *page.EventFrameStoppedLoading:
id, op = e.FrameID, frameStoppedLoading
// ignored events
case *page.EventFrameRequestedNavigation:
return
case *page.EventDomContentEventFired:
return
case *page.EventLoadEventFired:
return
case *page.EventFrameResized:
return
case *page.EventLifecycleEvent:
return
case *page.EventNavigatedWithinDocument:
return
default:
t.errf("unhandled page event %T", ev)
return
}
f := t.frames[id]
if f == nil {
// This can happen if a frame is attached or starts loading
// before it's ever navigated to. We won't have all the frame
// details just yet, but that's okay.
f = &cdp.Frame{ID: id}
t.frames[id] = f
}
f.Lock()
defer f.Unlock()
op(f)
}
// domEvent handles incoming DOM events.
func (t *Target) domEvent(ev interface{}) {
f := t.cur
var id cdp.NodeID
var op nodeOp
switch e := ev.(type) {
case *dom.EventSetChildNodes:
id, op = e.ParentID, setChildNodes(f.Nodes, e.Nodes)
case *dom.EventAttributeModified:
id, op = e.NodeID, attributeModified(e.Name, e.Value)
case *dom.EventAttributeRemoved:
id, op = e.NodeID, attributeRemoved(e.Name)
case *dom.EventInlineStyleInvalidated:
if len(e.NodeIds) == 0 {
return
}
id, op = e.NodeIds[0], inlineStyleInvalidated(e.NodeIds[1:])
case *dom.EventCharacterDataModified:
id, op = e.NodeID, characterDataModified(e.CharacterData)
case *dom.EventChildNodeCountUpdated:
id, op = e.NodeID, childNodeCountUpdated(e.ChildNodeCount)
case *dom.EventChildNodeInserted:
id, op = e.ParentNodeID, childNodeInserted(f.Nodes, e.PreviousNodeID, e.Node)
case *dom.EventChildNodeRemoved:
id, op = e.ParentNodeID, childNodeRemoved(f.Nodes, e.NodeID)
case *dom.EventShadowRootPushed:
id, op = e.HostID, shadowRootPushed(f.Nodes, e.Root)
case *dom.EventShadowRootPopped:
id, op = e.HostID, shadowRootPopped(f.Nodes, e.RootID)
case *dom.EventPseudoElementAdded:
id, op = e.ParentID, pseudoElementAdded(f.Nodes, e.PseudoElement)
case *dom.EventPseudoElementRemoved:
id, op = e.ParentID, pseudoElementRemoved(f.Nodes, e.PseudoElementID)
case *dom.EventDistributedNodesUpdated:
id, op = e.InsertionPointID, distributedNodesUpdated(e.DistributedNodes)
default:
t.errf("unhandled node event %T", ev)
return
}
n, ok := f.Nodes[id]
if !ok {
// Node ID has been invalidated. Nothing to do.
return
}
f.Lock()
defer f.Unlock()
op(n)
}
type TargetOption func(*Target)

9
testdata/iframe.html vendored
View File

@ -1,9 +0,0 @@
<!doctype html>
<html>
<head>
<title>page with an iframe</title>
</head>
<body>
<iframe src="form.html"></iframe>
</body>
</html>