commons_go/chromedp
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

start the chromedp v2 refactor

Browse Source 
First, we want all of the functionality in a single package; this means
collapsing whatever is useful into the root chromedp package.

The runner package is being replaced by the Allocator interface, with a
default implementation which starts browser processes.

The client package doesn't really have a place in the new design. The
context, allocator, and browser types will handle the connection with
each browser.

Finally, the new API is context-based, hence the addition of context.go.
The tests have been modified to build and run against the new API.
This commit is contained in:
Daniel Martí 2019-03-05 13:14:50 +00:00
parent 5aca12cc3e
commit 3d3bf22ccc
40 changed files with 1027 additions and 2820 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.travis.yml
View File

@ -1,14 +1,7 @@
language: go
go:
- 1.10.x
- 1.11.x
addons:
apt:
chrome: stable
before_install:
- go get github.com/mattn/goveralls golang.org/x/vgo
- 1.12.x
script:
- export CHROMEDP_TEST_RUNNER=google-chrome-stable
- export CHROMEDP_DISABLE_GPU=true
- vgo test -v -coverprofile=coverage.out
- goveralls -service=travis-ci -coverprofile=coverage.out
- true

26
_example/allocactor.go Normal file
View File

@ -0,0 +1,26 @@
package main
import (
"context"
"log"
"github.com/chromedp/chromedp"
)
func main() {
dockerAllocatorOpts := []chromedp.DockerAllocatorOption{}
ctxt, cancel := chromedp.NewAllocator(context.Background(), chromedp.WithDockerAllocator(dockerAllocatorOpts...))
defer cancel()
task1Context, cancel := chromedp.NewContext(ctxt)
defer cancel()
if err := chromedp.Run(task1Context, myTask()); err != nil {
log.Fatal(err)
}
}
func myTask() chromedp.Tasks {
return []chromedp.Action{}
}

36
_example/simple.go Normal file
View File

@ -0,0 +1,36 @@
package main
import (
"context"
"fmt"
"log"
"github.com/chromedp/chromedp"
)
func main() {
// create a new context
ctx, cancel := chromedp.NewContext(context.Background())
defer cancel()
// grab the title
var title string
if err := chromedp.Run(ctx, grabTitle(&title)); err != nil {
log.Fatal(err)
}
// print it
fmt.Println(title)
// ensure all resources are cleaned up
cancel()
chromedp.FromContext(ctx).Wait()
}
func grabTitle(title *string) chromedp.Tasks {
return []chromedp.Action{
chromedp.Navigate("https://github.com/"),
chromedp.WaitVisible("#start-of-content", chromedp.ByID),
chromedp.Title(title),
}
}

31
_example/tabs.go Normal file
View File

@ -0,0 +1,31 @@
package main
import (
"context"
"log"
"github.com/chromedp/chromedp"
)
func main() {
// first tab
ctx1, cancel := chromedp.NewContext(context.Background())
defer cancel()
// create new tab
ctx2, _ := chromedp.NewContext(ctx1)
// runs in first tab
if err := chromedp.Run(ctx1, myTask()); err != nil {
log.Fatal(err)
}
// runs in second tab
if err := chromedp.Run(ctx2, myTask()); err != nil {
log.Fatal(err)
}
}
func myTask() chromedp.Tasks {
return []chromedp.Action{}
}

255
allocate.go Normal file
View File

@ -0,0 +1,255 @@
package chromedp
import (
"bufio"
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"strings"
"sync"
)
type Allocator interface {
// Allocate creates a new browser from the pool. It can be cancelled via
// the provided context, at which point all the resources used by the
// browser (such as temporary directories) will be cleaned up.
Allocate(context.Context) (*Browser, error)
// Wait can be called after cancelling a pool's context, to block until
// all the pool's resources have been cleaned up.
Wait()
}
func NewAllocator(parent context.Context, opts ...AllocatorOption) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
c := &Context{}
for _, o := range opts {
o(&c.Allocator)
}
ctx = context.WithValue(ctx, contextKey{}, c)
return ctx, cancel
}
type AllocatorOption func(*Allocator)
func WithExecAllocator(opts ...ExecAllocatorOption) func(*Allocator) {
return func(p *Allocator) {
ep := &ExecAllocator{
initFlags: make(map[string]interface{}),
}
for _, o := range opts {
o(ep)
}
if ep.execPath == "" {
ep.execPath = findExecPath()
}
*p = ep
}
}
type ExecAllocatorOption func(*ExecAllocator)
type ExecAllocator struct {
execPath string
initFlags map[string]interface{}
wg sync.WaitGroup
}
func (p *ExecAllocator) Allocate(ctx context.Context) (*Browser, error) {
removeDir := false
var cmd *exec.Cmd
// TODO: figure out a nicer way to do this
flags := make(map[string]interface{})
for name, value := range p.initFlags {
flags[name] = value
}
dataDir, ok := flags["user-data-dir"].(string)
if !ok {
tempDir, err := ioutil.TempDir("", "chromedp-runner")
if err != nil {
return nil, err
}
flags["user-data-dir"] = tempDir
dataDir = tempDir
removeDir = true
}
p.wg.Add(1)
go func() {
<-ctx.Done()
// First wait for the process to be finished.
if cmd != nil {
cmd.Wait()
}
// Then delete the temporary user data directory, if needed.
if removeDir {
os.RemoveAll(dataDir)
}
p.wg.Done()
}()
flags["remote-debugging-port"] = "0"
args := []string{}
for name, value := range flags {
switch value := value.(type) {
case string:
args = append(args, fmt.Sprintf("--%s=%s", name, value))
case bool:
if value {
args = append(args, fmt.Sprintf("--%s", name))
}
default:
return nil, fmt.Errorf("invalid exec pool flag")
}
}
cmd = exec.CommandContext(ctx, p.execPath, args...)
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, err
}
// Pick up the browser's websocket URL from stderr.
wsURL := ""
scanner := bufio.NewScanner(stderr)
prefix := "DevTools listening on"
for scanner.Scan() {
line := scanner.Text()
if s := strings.TrimPrefix(line, prefix); s != line {
wsURL = strings.TrimSpace(s)
break
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
stderr.Close()
browser, err := NewBrowser(wsURL)
if err != nil {
return nil, err
}
browser.UserDataDir = dataDir
return browser, nil
}
func (p *ExecAllocator) Wait() {
p.wg.Wait()
}
func ExecPath(path string) ExecAllocatorOption {
return func(p *ExecAllocator) {
if fullPath, _ := exec.LookPath(path); fullPath != "" {
// Convert to an absolute path if possible, to avoid
// repeated LookPath calls in each Allocate.
path = fullPath
}
p.execPath = path
}
}
// findExecPath tries to find the Chrome browser somewhere in the current
// system. It performs a rather agressive search, which is the same in all
// systems. That may make it a bit slow, but it will only be run when creating a
// new ExecAllocator.
func findExecPath() string {
for _, path := range [...]string{
// Unix-like
"headless_shell",
"chromium",
"chromium-browser",
"google-chrome",
"google-chrome-stable",
"google-chrome-beta",
"google-chrome-unstable",
"/usr/bin/google-chrome",
// Windows
"chrome",
"chrome.exe", // in case PATHEXT is misconfigured
`C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`,
// Mac
`/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`,
} {
found, err := exec.LookPath(path)
if err == nil {
return found
}
}
// Fall back to something simple and sensible, to give a useful error
// message.
return "google-chrome"
}
// Flag is a generic command line option to pass a flag to Chrome. If the value
// is a string, it will be passed as --name=value. If it's a boolean, it will be
// passed as --name if value is true.
func Flag(name string, value interface{}) ExecAllocatorOption {
return func(p *ExecAllocator) {
p.initFlags[name] = value
}
}
// UserDataDir is the command line option to set the user data dir.
//
// Note: set this option to manually set the profile directory used by Chrome.
// When this is not set, then a default path will be created in the /tmp
// directory.
func UserDataDir(dir string) ExecAllocatorOption {
return Flag("user-data-dir", dir)
}
// ProxyServer is the command line option to set the outbound proxy server.
func ProxyServer(proxy string) ExecAllocatorOption {
return Flag("proxy-server", proxy)
}
// WindowSize is the command line option to set the initial window size.
func WindowSize(width, height int) ExecAllocatorOption {
return Flag("window-size", fmt.Sprintf("%d,%d", width, height))
}
// UserAgent is the command line option to set the default User-Agent
// header.
func UserAgent(userAgent string) ExecAllocatorOption {
return Flag("user-agent", userAgent)
}
// NoSandbox is the Chrome comamnd line option to disable the sandbox.
func NoSandbox(p *ExecAllocator) {
Flag("no-sandbox", true)(p)
}
// NoFirstRun is the Chrome comamnd line option to disable the first run
// dialog.
func NoFirstRun(p *ExecAllocator) {
Flag("no-first-run", true)(p)
}
// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the
// default browser check.
func NoDefaultBrowserCheck(p *ExecAllocator) {
Flag("no-default-browser-check", true)(p)
}
// Headless is the command line option to run in headless mode.
func Headless(p *ExecAllocator) {
Flag("headless", true)(p)
}
// DisableGPU is the command line option to disable the GPU process.
func DisableGPU(p *ExecAllocator) {
Flag("disable-gpu", true)(p)
}

43
allocate_test.go Normal file
View File

@ -0,0 +1,43 @@
package chromedp
import (
"context"
"os"
"testing"
)
func TestExecAllocator(t *testing.T) {
t.Parallel()
poolCtx, cancel := NewAllocator(context.Background(), WithExecAllocator(allocOpts...))
defer cancel()
// TODO: test that multiple child contexts are run in different
// processes and browsers.
taskCtx, cancel := NewContext(poolCtx)
defer cancel()
want := "insert"
var got string
if err := Run(taskCtx, Tasks{
Navigate(testdataDir + "/form.html"),
Text("#foo", &got, ByID),
}); err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("wanted %q, got %q", want, got)
}
tempDir := FromContext(taskCtx).browser.UserDataDir
pool := FromContext(taskCtx).Allocator
cancel()
pool.Wait()
if _, err := os.Lstat(tempDir); os.IsNotExist(err) {
return
}
t.Fatalf("temporary user data dir %q not deleted", tempDir)
}

122
browser.go Normal file
View File

@ -0,0 +1,122 @@
// Package chromedp is a high level Chrome DevTools Protocol client that
// simplifies driving browsers for scraping, unit testing, or profiling web
// pages using the CDP.
//
// chromedp requires no third-party dependencies, implementing the async Chrome
// DevTools Protocol entirely in Go.
package chromedp
import (
"context"
"log"
"sync/atomic"
"github.com/chromedp/cdproto"
"github.com/mailru/easyjson"
)
// Browser is the high-level Chrome DevTools Protocol browser manager, handling
// the browser process runner, WebSocket clients, associated targets, and
// network, page, and DOM events.
type Browser struct {
UserDataDir string
conn Transport
// next is the next message id.
next int64
// logging funcs
logf func(string, ...interface{})
errf func(string, ...interface{})
}
// NewBrowser creates a new browser.
func NewBrowser(urlstr string, opts ...BrowserOption) (*Browser, error) {
conn, err := Dial(ForceIP(urlstr))
if err != nil {
return nil, err
}
b := &Browser{
conn: conn,
logf: log.Printf,
}
// apply options
for _, o := range opts {
if err := o(b); err != nil {
return nil, err
}
}
// ensure errf is set
if b.errf == nil {
b.errf = func(s string, v ...interface{}) { b.logf("ERROR: "+s, v...) }
}
return b, nil
}
// Shutdown shuts down the browser.
func (b *Browser) Shutdown() error {
if b.conn != nil {
if err := b.send(cdproto.CommandBrowserClose, nil); err != nil {
b.errf("could not close browser: %v", err)
}
return b.conn.Close()
}
return nil
}
// send writes the supplied message and params.
func (b *Browser) send(method cdproto.MethodType, params easyjson.RawMessage) error {
msg := &cdproto.Message{
Method: method,
ID: atomic.AddInt64(&b.next, 1),
Params: params,
}
buf, err := msg.MarshalJSON()
if err != nil {
return err
}
return b.conn.Write(buf)
}
// sendToTarget writes the supplied message to the target.
func (b *Browser) sendToTarget(targetID string, method cdproto.MethodType, params easyjson.RawMessage) error {
return nil
}
// CreateContext creates a new browser context.
func (b *Browser) CreateContext() (context.Context, error) {
return nil, nil
}
// BrowserOption is a browser option.
type BrowserOption func(*Browser) error
// WithLogf is a browser option to specify a func to receive general logging.
func WithLogf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
b.logf = f
return nil
}
}
// WithErrorf is a browser option to specify a func to receive error logging.
func WithErrorf(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
b.errf = f
return nil
}
}
// WithConsolef is a browser option to specify a func to receive chrome log events.
//
// Note: NOT YET IMPLEMENTED.
func WithConsolef(f func(string, ...interface{})) BrowserOption {
return func(b *Browser) error {
return nil
}
}

435
chromedp.go
View File

@ -1,435 +0,0 @@
// Package chromedp is a high level Chrome DevTools Protocol client that
// simplifies driving browsers for scraping, unit testing, or profiling web
// pages using the CDP.
//
// chromedp requires no third-party dependencies, implementing the async Chrome
// DevTools Protocol entirely in Go.
package chromedp
import (
"context"
"errors"
"fmt"
"log"
"sync"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp/client"
"github.com/chromedp/chromedp/runner"
)
const (
// DefaultNewTargetTimeout is the default time to wait for a new target to
// be started.
DefaultNewTargetTimeout = 3 * time.Second
// DefaultCheckDuration is the default time to sleep between a check.
DefaultCheckDuration = 50 * time.Millisecond
// DefaultPoolStartPort is the default start port number.
DefaultPoolStartPort = 9000
// DefaultPoolEndPort is the default end port number.
DefaultPoolEndPort = 10000
)
// CDP is the high-level Chrome DevTools Protocol browser manager, handling the
// browser process runner, WebSocket clients, associated targets, and network,
// page, and DOM events.
type CDP struct {
// r is the chrome runner.
r *runner.Runner
// opts are command line options to pass to a created runner.
opts []runner.CommandLineOption
// watch is the channel for new client targets.
watch <-chan client.Target
// cur is the current active target's handler.
cur cdp.Executor
// handlers is the active handlers.
handlers []*TargetHandler
// handlerMap is the map of target IDs to its active handler.
handlerMap map[string]int
// logging funcs
logf, debugf, errf func(string, ...interface{})
sync.RWMutex
}
// New creates and starts a new CDP instance.
func New(ctxt context.Context, opts ...Option) (*CDP, error) {
c := &CDP{
handlers: make([]*TargetHandler, 0),
handlerMap: make(map[string]int),
logf: log.Printf,
debugf: func(string, ...interface{}) {},
errf: func(s string, v ...interface{}) { log.Printf("error: "+s, v...) },
}
// apply options
for _, o := range opts {
if err := o(c); err != nil {
return nil, err
}
}
// check for supplied runner, if none then create one
if c.r == nil && c.watch == nil {
var err error
c.r, err = runner.Run(ctxt, c.opts...)
if err != nil {
return nil, err
}
}
// watch handlers
if c.watch == nil {
c.watch = c.r.Client().WatchPageTargets(ctxt)
}
go func() {
for t := range c.watch {
if t == nil {
return
}
go c.AddTarget(ctxt, t)
}
}()
// TODO: fix this
timeout := time.After(defaultNewTargetTimeout)
// wait until at least one target active
for {
select {
default:
c.RLock()
exists := c.cur != nil
c.RUnlock()
if exists {
return c, nil
}
// TODO: fix this
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return nil, ctxt.Err()
case <-timeout:
return nil, errors.New("timeout waiting for initial target")
}
}
}
// AddTarget adds a target using the supplied context.
func (c *CDP) AddTarget(ctxt context.Context, t client.Target) {
c.Lock()
defer c.Unlock()
// create target manager
h, err := NewTargetHandler(t, c.logf, c.debugf, c.errf)
if err != nil {
c.errf("could not create handler for %s: %v", t, err)
return
}
// run
if err := h.Run(ctxt); err != nil {
c.errf("could not start handler for %s: %v", t, err)
return
}
// add to active handlers
c.handlers = append(c.handlers, h)
c.handlerMap[t.GetID()] = len(c.handlers) - 1
if c.cur == nil {
c.cur = h
}
}
// Wait waits for the Chrome runner to terminate.
func (c *CDP) Wait() error {
c.RLock()
r := c.r
c.RUnlock()
if r != nil {
return r.Wait()
}
return nil
}
// Shutdown closes all Chrome page handlers.
func (c *CDP) Shutdown(ctxt context.Context, opts ...client.Option) error {
c.RLock()
defer c.RUnlock()
if c.r != nil {
return c.r.Shutdown(ctxt, opts...)
}
return nil
}
// ListTargets returns the target IDs of the managed targets.
func (c *CDP) ListTargets() []string {
c.RLock()
defer c.RUnlock()
i, targets := 0, make([]string, len(c.handlers))
for k := range c.handlerMap {
targets[i] = k
i++
}
return targets
}
// GetHandlerByIndex retrieves the domains manager for the specified index.
func (c *CDP) GetHandlerByIndex(i int) cdp.Executor {
c.RLock()
defer c.RUnlock()
if i < 0 || i >= len(c.handlers) {
return nil
}
return c.handlers[i]
}
// GetHandlerByID retrieves the domains manager for the specified target ID.
func (c *CDP) GetHandlerByID(id string) cdp.Executor {
c.RLock()
defer c.RUnlock()
if i, ok := c.handlerMap[id]; ok {
return c.handlers[i]
}
return nil
}
// SetHandler sets the active handler to the target with the specified index.
func (c *CDP) SetHandler(i int) error {
c.Lock()
defer c.Unlock()
if i < 0 || i >= len(c.handlers) {
return fmt.Errorf("no handler associated with target index %d", i)
}
c.cur = c.handlers[i]
return nil
}
// SetHandlerByID sets the active target to the target with the specified id.
func (c *CDP) SetHandlerByID(id string) error {
c.Lock()
defer c.Unlock()
if i, ok := c.handlerMap[id]; ok {
c.cur = c.handlers[i]
return nil
}
return fmt.Errorf("no handler associated with target id %s", id)
}
// newTarget creates a new target using supplied context and options, returning
// the id of the created target only after the target has been started for
// monitoring.
func (c *CDP) newTarget(ctxt context.Context, opts ...client.Option) (string, error) {
c.RLock()
cl := c.r.Client(opts...)
c.RUnlock()
// new page target
t, err := cl.NewPageTarget(ctxt)
if err != nil {
return "", err
}
timeout := time.After(DefaultNewTargetTimeout)
for {
select {
default:
var ok bool
id := t.GetID()
c.RLock()
_, ok = c.handlerMap[id]
c.RUnlock()
if ok {
return id, nil
}
time.Sleep(DefaultCheckDuration)
case <-ctxt.Done():
return "", ctxt.Err()
case <-timeout:
return "", errors.New("timeout waiting for new target to be available")
}
}
}
// SetTarget is an action that sets the active Chrome handler to the specified
// index i.
func (c *CDP) SetTarget(i int) Action {
return ActionFunc(func(context.Context, cdp.Executor) error {
return c.SetHandler(i)
})
}
// SetTargetByID is an action that sets the active Chrome handler to the handler
// associated with the specified id.
func (c *CDP) SetTargetByID(id string) Action {
return ActionFunc(func(context.Context, cdp.Executor) error {
return c.SetHandlerByID(id)
})
}
// NewTarget is an action that creates a new Chrome target, and sets it as the
// active target.
func (c *CDP) NewTarget(id *string, opts ...client.Option) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
n, err := c.newTarget(ctxt, opts...)
if err != nil {
return err
}
if id != nil {
*id = n
}
return nil
})
}
// CloseByIndex closes the Chrome target with specified index i.
func (c *CDP) CloseByIndex(i int) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
return nil
})
}
// CloseByID closes the Chrome target with the specified id.
func (c *CDP) CloseByID(id string) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
return nil
})
}
// Run executes the action against the current target using the supplied
// context.
func (c *CDP) Run(ctxt context.Context, a Action) error {
c.RLock()
cur := c.cur
c.RUnlock()
return a.Do(ctxt, cur)
}
// Option is a Chrome DevTools Protocol option.
type Option func(*CDP) error
// WithRunner is a CDP option to specify the underlying Chrome runner to
// monitor for page handlers.
func WithRunner(r *runner.Runner) Option {
return func(c *CDP) error {
c.r = r
return nil
}
}
// WithTargets is a CDP option to specify the incoming targets to monitor for
// page handlers.
func WithTargets(watch <-chan client.Target) Option {
return func(c *CDP) error {
c.watch = watch
return nil
}
}
// WithClient is a CDP option to use the incoming targets from a client.
func WithClient(ctxt context.Context, cl *client.Client) Option {
return func(c *CDP) error {
return WithTargets(cl.WatchPageTargets(ctxt))(c)
}
}
// WithURL is a CDP option to use a client with the specified URL.
func WithURL(ctxt context.Context, urlstr string) Option {
return func(c *CDP) error {
return WithClient(ctxt, client.New(client.URL(urlstr)))(c)
}
}
// WithRunnerOptions is a CDP option to specify the options to pass to a newly
// created Chrome process runner.
func WithRunnerOptions(opts ...runner.CommandLineOption) Option {
return func(c *CDP) error {
c.opts = opts
return nil
}
}
// WithLogf is a CDP option to specify a func to receive general logging.
func WithLogf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.logf = f
return nil
}
}
// WithDebugf is a CDP option to specify a func to receive debug logging (ie,
// protocol information).
func WithDebugf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.debugf = f
return nil
}
}
// WithErrorf is a CDP option to specify a func to receive error logging.
func WithErrorf(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.errf = f
return nil
}
}
// WithLog is a CDP option that sets the logging, debugging, and error funcs to
// f.
func WithLog(f func(string, ...interface{})) Option {
return func(c *CDP) error {
c.logf, c.debugf, c.errf = f, f, f
return nil
}
}
// WithConsolef is a CDP option to specify a func to receive chrome log events.
//
// Note: NOT YET IMPLEMENTED.
func WithConsolef(f func(string, ...interface{})) Option {
return func(c *CDP) error {
return nil
}
}
var (
// defaultNewTargetTimeout is the default target timeout -- used by
// testing.
defaultNewTargetTimeout = DefaultNewTargetTimeout
)

108
chromedp_test.go
View File

@ -2,117 +2,71 @@ package chromedp
import (
"context"
"log"
"fmt"
"os"
"path"
"testing"
"time"
"github.com/chromedp/chromedp/runner"
)
var (
pool *Pool
testdataDir string
defaultContext, defaultCancel = context.WithCancel(context.Background())
allocCtx context.Context
cliOpts = []runner.CommandLineOption{
runner.NoDefaultBrowserCheck,
runner.NoFirstRun,
allocOpts = []ExecAllocatorOption{
NoFirstRun,
NoDefaultBrowserCheck,
Headless,
}
)
func testAllocate(t *testing.T, path string) *Res {
c, err := pool.Allocate(defaultContext, cliOpts...)
if err != nil {
t.Fatalf("could not allocate from pool: %v", err)
func testAllocate(t *testing.T, path string) (_ context.Context, cancel func()) {
ctx, cancel := NewContext(allocCtx)
if err := Run(ctx, Navigate(testdataDir+"/"+path)); err != nil {
t.Fatal(err)
}
err = WithLogf(t.Logf)(c.c)
if err != nil {
t.Fatalf("could not set logf: %v", err)
}
//if err := WithLogf(t.Logf)(c.c); err != nil {
// t.Fatalf("could not set logf: %v", err)
//}
//if err := WithDebugf(t.Logf)(c.c); err != nil {
// t.Fatalf("could not set debugf: %v", err)
//}
//if err := WithErrorf(t.Errorf)(c.c); err != nil {
// t.Fatalf("could not set errorf: %v", err)
//}
err = WithDebugf(t.Logf)(c.c)
if err != nil {
t.Fatalf("could not set debugf: %v", err)
}
err = WithErrorf(t.Errorf)(c.c)
if err != nil {
t.Fatalf("could not set errorf: %v", err)
}
h := c.c.GetHandlerByIndex(0)
th, ok := h.(*TargetHandler)
if !ok {
t.Fatalf("handler is invalid type")
}
th.logf, th.debugf = t.Logf, t.Logf
th.errf = func(s string, v ...interface{}) {
t.Logf("TARGET HANDLER ERROR: "+s, v...)
}
if path != "" {
err = c.Run(defaultContext, Navigate(testdataDir+"/"+path))
if err != nil {
t.Fatalf("could not navigate to testdata/%s: %v", path, err)
}
}
return c
return ctx, cancel
}
func TestMain(m *testing.M) {
var err error
wd, err := os.Getwd()
if err != nil {
log.Fatalf("could not get working directory: %v", err)
os.Exit(1)
panic(fmt.Sprintf("could not get working directory: %v", err))
}
testdataDir = "file://" + path.Join(wd, "testdata")
// its worth noting that newer versions of chrome (64+) run much faster
// it's worth noting that newer versions of chrome (64+) run much faster
// than older ones -- same for headless_shell ...
execPath := os.Getenv("CHROMEDP_TEST_RUNNER")
if execPath == "" {
execPath = runner.LookChromeNames("headless_shell")
if execPath := os.Getenv("CHROMEDP_TEST_RUNNER"); execPath != "" {
allocOpts = append(allocOpts, ExecPath(execPath))
}
cliOpts = append(cliOpts, runner.ExecPath(execPath))
// not explicitly needed to be set, as this vastly speeds up unit tests
if noSandbox := os.Getenv("CHROMEDP_NO_SANDBOX"); noSandbox != "false" {
cliOpts = append(cliOpts, runner.NoSandbox)
allocOpts = append(allocOpts, NoSandbox)
}
// must be explicitly set, as disabling gpu slows unit tests
if disableGPU := os.Getenv("CHROMEDP_DISABLE_GPU"); disableGPU != "" && disableGPU != "false" {
cliOpts = append(cliOpts, runner.DisableGPU)
allocOpts = append(allocOpts, DisableGPU)
}
if targetTimeout := os.Getenv("CHROMEDP_TARGET_TIMEOUT"); targetTimeout != "" {
defaultNewTargetTimeout, _ = time.ParseDuration(targetTimeout)
}
if defaultNewTargetTimeout == 0 {
defaultNewTargetTimeout = 30 * time.Second
}
//pool, err = NewPool(PoolLog(log.Printf, log.Printf, log.Printf))
pool, err = NewPool()
if err != nil {
log.Fatal(err)
}
ctx, cancel := NewAllocator(context.Background(), WithExecAllocator(allocOpts...))
allocCtx = ctx
code := m.Run()
defaultCancel()
err = pool.Shutdown()
if err != nil {
log.Fatal(err)
}
cancel()
FromContext(ctx).Wait()
os.Exit(code)
}

46
client/chrome.go
View File

@ -1,46 +0,0 @@
package client
import "fmt"
//go:generate easyjson -omit_empty -output_filename easyjson.go chrome.go
// Chrome holds connection information for a Chrome target.
//
//easyjson:json
type Chrome struct {
Description string `json:"description,omitempty"`
DevtoolsURL string `json:"devtoolsFrontendUrl,omitempty"`
ID string `json:"id,omitempty"`
Title string `json:"title,omitempty"`
Type TargetType `json:"type,omitempty"`
URL string `json:"url,omitempty"`
WebsocketURL string `json:"webSocketDebuggerUrl,omitempty"`
FaviconURL string `json:"faviconURL,omitempty"`
}
// String satisfies the stringer interface.
func (c Chrome) String() string {
return fmt.Sprintf("[%s]: %q", c.ID, c.Title)
}
// GetID returns the target ID.
func (c *Chrome) GetID() string {
return c.ID
}
// GetType returns the target type.
func (c *Chrome) GetType() TargetType {
return c.Type
}
// GetDevtoolsURL returns the devtools frontend target URL, satisfying the
// domains.Target interface.
func (c *Chrome) GetDevtoolsURL() string {
return c.DevtoolsURL
}
// GetWebsocketURL provides the websocket URL for the target, satisfying the
// domains.Target interface.
func (c *Chrome) GetWebsocketURL() string {
return c.WebsocketURL
}

340
client/client.go
View File

@ -1,340 +0,0 @@
// Package client provides the low level Chrome DevTools Protocol client.
package client
//go:generate go run gen.go
import (
"context"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"
"sync"
"time"
"github.com/mailru/easyjson"
)
const (
// DefaultEndpoint is the default endpoint to connect to.
DefaultEndpoint = "http://localhost:9222/json"
// DefaultWatchInterval is the default check duration.
DefaultWatchInterval = 100 * time.Millisecond
// DefaultWatchTimeout is the default watch timeout.
DefaultWatchTimeout = 5 * time.Second
)
// Error is a client error.
type Error string
// Error satisfies the error interface.
func (err Error) Error() string {
return string(err)
}
const (
// ErrUnsupportedProtocolType is the unsupported protocol type error.
ErrUnsupportedProtocolType Error = "unsupported protocol type"
// ErrUnsupportedProtocolVersion is the unsupported protocol version error.
ErrUnsupportedProtocolVersion Error = "unsupported protocol version"
)
// Target is the common interface for a Chrome DevTools Protocol target.
type Target interface {
String() string
GetID() string
GetType() TargetType
GetDevtoolsURL() string
GetWebsocketURL() string
}
// Client is a Chrome DevTools Protocol client.
type Client struct {
url string
check time.Duration
timeout time.Duration
ver, typ string
rw sync.RWMutex
}
// New creates a new Chrome DevTools Protocol client.
func New(opts ...Option) *Client {
c := &Client{
url: DefaultEndpoint,
check: DefaultWatchInterval,
timeout: DefaultWatchTimeout,
}
// apply opts
for _, o := range opts {
o(c)
}
return c
}
// doReq executes a request.
func (c *Client) doReq(ctxt context.Context, action string, v interface{}) error {
// create request
req, err := http.NewRequest("GET", c.url+"/"+action, nil)
if err != nil {
return err
}
req = req.WithContext(ctxt)
cl := &http.Client{}
// execute
res, err := cl.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if v != nil {
// load body
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
// unmarshal
if z, ok := v.(easyjson.Unmarshaler); ok {
return easyjson.Unmarshal(body, z)
}
return json.Unmarshal(body, v)
}
return nil
}
// ListTargets returns a list of all targets.
func (c *Client) ListTargets(ctxt context.Context) ([]Target, error) {
var err error
var l []json.RawMessage
if err = c.doReq(ctxt, "list", &l); err != nil {
return nil, err
}
t := make([]Target, len(l))
for i, v := range l {
t[i], err = c.newTarget(ctxt, v)
if err != nil {
return nil, err
}
}
return t, nil
}
// ListTargetsWithType returns a list of Targets with the specified target
// type.
func (c *Client) ListTargetsWithType(ctxt context.Context, typ TargetType) ([]Target, error) {
var err error
targets, err := c.ListTargets(ctxt)
if err != nil {
return nil, err
}
var ret []Target
for _, t := range targets {
if t.GetType() == typ {
ret = append(ret, t)
}
}
return ret, nil
}
// ListPageTargets lists the available Page targets.
func (c *Client) ListPageTargets(ctxt context.Context) ([]Target, error) {
return c.ListTargetsWithType(ctxt, Page)
}
var browserRE = regexp.MustCompile(`(?i)^(chrome|chromium|microsoft edge|safari)`)
// loadProtocolInfo loads the protocol information from the remote URL.
func (c *Client) loadProtocolInfo(ctxt context.Context) (string, string, error) {
c.rw.Lock()
defer c.rw.Unlock()
if c.ver == "" || c.typ == "" {
v, err := c.VersionInfo(ctxt)
if err != nil {
return "", "", err
}
if m := browserRE.FindAllStringSubmatch(v["Browser"], -1); len(m) != 0 {
c.typ = strings.ToLower(m[0][0])
}
c.ver = v["Protocol-Version"]
}
return c.ver, c.typ, nil
}
// newTarget creates a new target.
func (c *Client) newTarget(ctxt context.Context, buf []byte) (Target, error) {
var err error
ver, typ, err := c.loadProtocolInfo(ctxt)
if err != nil {
return nil, err
}
if ver != "1.1" && ver != "1.2" && ver != "1.3" {
return nil, ErrUnsupportedProtocolVersion
}
switch typ {
case "chrome", "chromium", "microsoft edge", "safari", "":
x := new(Chrome)
if buf != nil {
if err = easyjson.Unmarshal(buf, x); err != nil {
return nil, err
}
}
return x, nil
}
return nil, ErrUnsupportedProtocolType
}
// NewPageTargetWithURL creates a new page target with the specified url.
func (c *Client) NewPageTargetWithURL(ctxt context.Context, urlstr string) (Target, error) {
var err error
t, err := c.newTarget(ctxt, nil)
if err != nil {
return nil, err
}
u := "new"
if urlstr != "" {
u += "?" + urlstr
}
if err = c.doReq(ctxt, u, t); err != nil {
return nil, err
}
return t, nil
}
// NewPageTarget creates a new page target.
func (c *Client) NewPageTarget(ctxt context.Context) (Target, error) {
return c.NewPageTargetWithURL(ctxt, "")
}
// ActivateTarget activates a target.
func (c *Client) ActivateTarget(ctxt context.Context, t Target) error {
return c.doReq(ctxt, "activate/"+t.GetID(), nil)
}
// CloseTarget activates a target.
func (c *Client) CloseTarget(ctxt context.Context, t Target) error {
return c.doReq(ctxt, "close/"+t.GetID(), nil)
}
// VersionInfo returns information about the remote debugging protocol.
func (c *Client) VersionInfo(ctxt context.Context) (map[string]string, error) {
v := make(map[string]string)
if err := c.doReq(ctxt, "version", &v); err != nil {
return nil, err
}
return v, nil
}
// WatchPageTargets watches for new page targets.
func (c *Client) WatchPageTargets(ctxt context.Context) <-chan Target {
ch := make(chan Target)
go func() {
defer close(ch)
encountered := make(map[string]bool)
check := func() error {
targets, err := c.ListPageTargets(ctxt)
if err != nil {
return err
}
for _, t := range targets {
if !encountered[t.GetID()] {
ch <- t
}
encountered[t.GetID()] = true
}
return nil
}
var err error
lastGood := time.Now()
for {
err = check()
if err == nil {
lastGood = time.Now()
} else if time.Now().After(lastGood.Add(c.timeout)) {
return
}
select {
case <-time.After(c.check):
continue
case <-ctxt.Done():
return
}
}
}()
return ch
}
// Option is a Chrome DevTools Protocol client option.
type Option func(*Client)
// URL is a client option to specify the remote Chrome DevTools Protocol
// instance to connect to.
func URL(urlstr string) Option {
return func(c *Client) {
// since chrome 66+, dev tools requires the host name to be either an
// IP address, or "localhost"
if strings.HasPrefix(strings.ToLower(urlstr), "http://") {
host, port, path := urlstr[7:], "", ""
if i := strings.Index(host, "/"); i != -1 {
host, path = host[:i], host[i:]
}
if i := strings.Index(host, ":"); i != -1 {
host, port = host[:i], host[i:]
}
if addr, err := net.ResolveIPAddr("ip", host); err == nil {
urlstr = "http://" + addr.IP.String() + port + path
}
}
c.url = urlstr
}
}
// WatchInterval is a client option that specifies the check interval duration.
func WatchInterval(check time.Duration) Option {
return func(c *Client) {
c.check = check
}
}
// WatchTimeout is a client option that specifies the watch timeout duration.
func WatchTimeout(timeout time.Duration) Option {
return func(c *Client) {
c.timeout = timeout
}
}

174
client/easyjson.go
View File

@ -1,174 +0,0 @@
// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT.
package client
import (
json "encoding/json"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// suppress unused package warning
var (
_ *json.RawMessage
_ *jlexer.Lexer
_ *jwriter.Writer
_ easyjson.Marshaler
)
func easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(in *jlexer.Lexer, out *Chrome) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "description":
out.Description = string(in.String())
case "devtoolsFrontendUrl":
out.DevtoolsURL = string(in.String())
case "id":
out.ID = string(in.String())
case "title":
out.Title = string(in.String())
case "type":
(out.Type).UnmarshalEasyJSON(in)
case "url":
out.URL = string(in.String())
case "webSocketDebuggerUrl":
out.WebsocketURL = string(in.String())
case "faviconURL":
out.FaviconURL = string(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(out *jwriter.Writer, in Chrome) {
out.RawByte('{')
first := true
_ = first
if in.Description != "" {
const prefix string = ",\"description\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Description))
}
if in.DevtoolsURL != "" {
const prefix string = ",\"devtoolsFrontendUrl\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.DevtoolsURL))
}
if in.ID != "" {
const prefix string = ",\"id\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.ID))
}
if in.Title != "" {
const prefix string = ",\"title\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Title))
}
if in.Type != "" {
const prefix string = ",\"type\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
(in.Type).MarshalEasyJSON(out)
}
if in.URL != "" {
const prefix string = ",\"url\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.URL))
}
if in.WebsocketURL != "" {
const prefix string = ",\"webSocketDebuggerUrl\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.WebsocketURL))
}
if in.FaviconURL != "" {
const prefix string = ",\"faviconURL\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.FaviconURL))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v Chrome) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v Chrome) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpChromedpClient(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *Chrome) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *Chrome) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpChromedpClient(l, v)
}

145
client/gen.go
View File

@ -1,145 +0,0 @@
// +build ignore
package main
import (
"encoding/base64"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os/exec"
"regexp"
"sort"
"github.com/knq/snaker"
)
const (
// chromiumSrc is the base chromium source repo location
chromiumSrc = "https://chromium.googlesource.com/chromium/src"
// devtoolsHTTPClientCc contains the target_type names.
devtoolsHTTPClientCc = chromiumSrc + "/+/master/chrome/test/chromedriver/chrome/devtools_http_client.cc?format=TEXT"
)
var (
flagOut = flag.String("out", "targettype.go", "out file")
)
func main() {
flag.Parse()
if err := run(); err != nil {
log.Fatal(err)
}
}
var typeAsStringRE = regexp.MustCompile(`type_as_string\s+==\s+"([^"]+)"`)
// run executes the generator.
func run() error {
var err error
// grab source
buf, err := grab(devtoolsHTTPClientCc)
if err != nil {
return err
}
// find names
matches := typeAsStringRE.FindAllStringSubmatch(string(buf), -1)
names := make([]string, len(matches))
for i, m := range matches {
names[i] = m[1]
}
sort.Strings(names)
// process names
var constVals, decodeVals string
for _, n := range names {
name := snaker.SnakeToCamelIdentifier(n)
constVals += fmt.Sprintf("%s TargetType = \"%s\"\n", name, n)
decodeVals += fmt.Sprintf("case %s:\n*tt=%s\n", name, name)
}
if err = ioutil.WriteFile(*flagOut, []byte(fmt.Sprintf(targetTypeSrc, constVals, decodeVals)), 0644); err != nil {
return err
}
return exec.Command("gofmt", "-w", "-s", *flagOut).Run()
}
// grab retrieves a file from the chromium source code.
func grab(path string) ([]byte, error) {
res, err := http.Get(path)
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
buf, err := base64.StdEncoding.DecodeString(string(body))
if err != nil {
return nil, err
}
return buf, nil
}
const (
targetTypeSrc = `package client
// Code generated by gen.go. DO NOT EDIT.
import (
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// TargetType are the types of targets available in Chrome.
type TargetType string
// TargetType values.
const (
%s
)
// String satisfies stringer.
func (tt TargetType) String() string {
return string(tt)
}
// MarshalEasyJSON satisfies easyjson.Marshaler.
func (tt TargetType) MarshalEasyJSON(out *jwriter.Writer) {
out.String(string(tt))
}
// MarshalJSON satisfies json.Marshaler.
func (tt TargetType) MarshalJSON() ([]byte, error) {
return easyjson.Marshal(tt)
}
// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
z := TargetType(in.String())
switch z {
%s
default:
*tt = z
}
}
// UnmarshalJSON satisfies json.Unmarshaler.
func (tt *TargetType) UnmarshalJSON(buf []byte) error {
return easyjson.Unmarshal(buf, tt)
}
`
)

79
client/targettype.go
View File

@ -1,79 +0,0 @@
package client
// Code generated by gen.go. DO NOT EDIT.
import (
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// TargetType are the types of targets available in Chrome.
type TargetType string
// TargetType values.
const (
App TargetType = "app"
BackgroundPage TargetType = "background_page"
Browser TargetType = "browser"
External TargetType = "external"
Iframe TargetType = "iframe"
Other TargetType = "other"
Page TargetType = "page"
ServiceWorker TargetType = "service_worker"
SharedWorker TargetType = "shared_worker"
Webview TargetType = "webview"
Worker TargetType = "worker"
)
// String satisfies stringer.
func (tt TargetType) String() string {
return string(tt)
}
// MarshalEasyJSON satisfies easyjson.Marshaler.
func (tt TargetType) MarshalEasyJSON(out *jwriter.Writer) {
out.String(string(tt))
}
// MarshalJSON satisfies json.Marshaler.
func (tt TargetType) MarshalJSON() ([]byte, error) {
return easyjson.Marshal(tt)
}
// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
z := TargetType(in.String())
switch z {
case App:
*tt = App
case BackgroundPage:
*tt = BackgroundPage
case Browser:
*tt = Browser
case External:
*tt = External
case Iframe:
*tt = Iframe
case Other:
*tt = Other
case Page:
*tt = Page
case ServiceWorker:
*tt = ServiceWorker
case SharedWorker:
*tt = SharedWorker
case Webview:
*tt = Webview
case Worker:
*tt = Worker
default:
*tt = z
}
}
// UnmarshalJSON satisfies json.Unmarshaler.
func (tt *TargetType) UnmarshalJSON(buf []byte) error {
return easyjson.Unmarshal(buf, tt)
}

41
client/transport.go → conn.go
View File

@ -1,7 +1,9 @@
package client
package chromedp
import (
"io"
"net"
"strings"
"github.com/gorilla/websocket"
)
@ -26,7 +28,7 @@ type Conn struct {
*websocket.Conn
}
// Read reads the next websocket message.
// Read reads the next message.
func (c *Conn) Read() ([]byte, error) {
_, buf, err := c.ReadMessage()
if err != nil {
@ -35,25 +37,18 @@ func (c *Conn) Read() ([]byte, error) {
return buf, nil
}
// Write writes a websocket message.
// Write writes a message.
func (c *Conn) Write(buf []byte) error {
return c.WriteMessage(websocket.TextMessage, buf)
}
// Dial dials the specified target's websocket URL.
//
// Note: uses gorilla/websocket.
func Dial(urlstr string, opts ...DialOption) (Transport, error) {
// Dial dials the specified websocket URL using gorilla/websocket.
func Dial(urlstr string) (*Conn, error) {
d := &websocket.Dialer{
ReadBufferSize: DefaultReadBufferSize,
WriteBufferSize: DefaultWriteBufferSize,
}
// apply opts
for _, o := range opts {
o(d)
}
// connect
conn, _, err := d.Dial(urlstr, nil)
if err != nil {
@ -63,5 +58,23 @@ func Dial(urlstr string, opts ...DialOption) (Transport, error) {
return &Conn{conn}, nil
}
// DialOption is a dial option.
type DialOption func(*websocket.Dialer)
// ForceIP forces the host component in urlstr to be an IP address.
//
// Since Chrome 66+, Chrome DevTools Protocol clients connecting to a browser
// must send the "Host:" header as either an IP address, or "localhost".
func ForceIP(urlstr string) string {
if i := strings.Index(urlstr, "://"); i != -1 {
scheme := urlstr[:i+3]
host, port, path := urlstr[len(scheme)+3:], "", ""
if i := strings.Index(host, "/"); i != -1 {
host, path = host[:i], host[i:]
}
if i := strings.Index(host, ":"); i != -1 {
host, port = host[:i], host[i:]
}
if addr, err := net.ResolveIPAddr("ip", host); err == nil {
urlstr = scheme + addr.IP.String() + port + path
}
}
return urlstr
}

115
context.go Normal file
View File

@ -0,0 +1,115 @@
package chromedp
import (
"context"
"encoding/json"
"net/http"
)
// Executor
type Executor interface {
Execute(context.Context, string, json.Marshaler, json.Unmarshaler) error
}
// Context
type Context struct {
Allocator Allocator
browser *Browser
handler *TargetHandler
logf func(string, ...interface{})
errf func(string, ...interface{})
}
// Wait can be called after cancelling the context containing Context, to block
// until all the underlying resources have been cleaned up.
func (c *Context) Wait() {
if c.Allocator != nil {
c.Allocator.Wait()
}
}
// NewContext creates a browser context using the parent context.
func NewContext(parent context.Context, opts ...ContextOption) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
c := &Context{}
if pc := FromContext(parent); pc != nil {
c.Allocator = pc.Allocator
}
for _, o := range opts {
o(c)
}
if c.Allocator == nil {
WithExecAllocator(
NoFirstRun,
NoDefaultBrowserCheck,
Headless,
)(&c.Allocator)
}
ctx = context.WithValue(ctx, contextKey{}, c)
return ctx, cancel
}
type contextKey struct{}
// FromContext creates a new browser context from the provided context.
func FromContext(ctx context.Context) *Context {
c, _ := ctx.Value(contextKey{}).(*Context)
return c
}
// Run runs the action against the provided browser context.
func Run(ctx context.Context, action Action) error {
c := FromContext(ctx)
if c == nil || c.Allocator == nil {
return ErrInvalidContext
}
if c.browser == nil {
browser, err := c.Allocator.Allocate(ctx)
if err != nil {
return err
}
c.browser = browser
}
if c.handler == nil {
if err := c.newHandler(ctx); err != nil {
return err
}
}
return action.Do(ctx, c.handler)
}
func (c *Context) newHandler(ctx context.Context) error {
// TODO: add RemoteAddr() to the Transport interface?
conn := c.browser.conn.(*Conn).Conn
addr := conn.RemoteAddr()
url := "http://" + addr.String() + "/json/new"
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
var wurl withWebsocketURL
if err := json.NewDecoder(resp.Body).Decode(&wurl); err != nil {
return err
}
c.handler, err = NewTargetHandler(wurl.WebsocketURL)
if err != nil {
return err
}
if err := c.handler.Run(ctx); err != nil {
return err
}
return nil
}
type withWebsocketURL struct {
WebsocketURL string `json:"webSocketDebuggerUrl"`
}
// ContextOption
type ContextOption func(*Context)

34
contrib/meta.sh
View File

@ -1,34 +0,0 @@
#!/bin/bash
SRC=$(realpath $(cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../)
pushd $SRC &> /dev/null
gometalinter \
--disable=aligncheck \
--enable=misspell \
--enable=gofmt \
--deadline=100s \
--cyclo-over=25 \
--sort=path \
--exclude='\(defer (.+?)\)\) \(errcheck\)$' \
--exclude='/easyjson\.go.*(passes|copies) lock' \
--exclude='/easyjson\.go.*ineffectual assignment' \
--exclude='/easyjson\.go.*unnecessary conversion' \
--exclude='/easyjson\.go.*this value of key is never used' \
--exclude='/easyjson\.go.*\((gocyclo|golint|goconst|staticcheck)\)$' \
--exclude='^cdp/.*Potential hardcoded credentials' \
--exclude='^cdp/cdp\.go.*UnmarshalEasyJSON.*\(gocyclo\)$' \
--exclude='^cdp/cdputil/cdputil\.go.*UnmarshalMessage.*\(gocyclo\)$' \
--exclude='^cmd/chromedp-gen/.*\((gocyclo|interfacer)\)$' \
--exclude='^cmd/chromedp-proxy/main\.go.*\(gas\)$' \
--exclude='^cmd/chromedp-gen/fixup/fixup\.go.*\(goconst\)$' \
--exclude='^cmd/chromedp-gen/internal/enum\.go.*unreachable' \
--exclude='^cmd/chromedp-gen/(main|domain-gen)\.go.*\(gas\)$' \
--exclude='^examples/[a-z]+/main\.go.*\(errcheck\)$' \
--exclude='^kb/gen\.go.*\((gas|vet)\)$' \
--exclude='^runner/.*\(gas\)$' \
--exclude='^handler\.go.*cmd can be easyjson\.Marshaler' \
./...
popd &> /dev/null

10
contrib/start.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
TMP=$(mktemp -d /tmp/google-chrome.XXXXX)
google-chrome \
--user-data-dir=$TMP \
--remote-debugging-port=9222 \
--no-first-run \
--no-default-browser-check \
about:blank

14
contrib/stats.sh
View File

@ -1,14 +0,0 @@
#!/bin/bash
BASE=$(realpath $(cd -P $GOPATH/src/github.com/chromedp && pwd))
FILES=$(find $BASE/{chromedp*,goquery,examples} -type f -iname \*.go -not -iname \*.qtpl.go -print0|wc -l --files0-from=-|head -n -1)$'\n'
AUTOG=$(find $BASE/cdproto/ -type f -iname \*.go -not -iname \*easyjson\* -print0|wc -l --files0-from=-|head -n -1)
if [ "$1" != "--total" ]; then
echo -e "code:\n$FILES\n\ngenerated:\n$AUTOG"
else
echo "code: $(awk '{s+=$1} END {print s}' <<< "$FILES")"
echo "generated: $(awk '{s+=$1} END {print s}' <<< "$AUTOG")"
fi

3
errors.go
View File

@ -39,4 +39,7 @@ const (
// ErrInvalidHandler is the invalid handler error.
ErrInvalidHandler Error = "invalid handler"
// ErrInvalidContext is the invalid context error.
ErrInvalidContext Error = "invalid context"
)

39
handler.go
View File

@ -20,13 +20,11 @@ import (
"github.com/chromedp/cdproto/log"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/chromedp/client"
)
// TargetHandler manages a Chrome DevTools Protocol target.
type TargetHandler struct {
conn client.Transport
conn Transport
// frames is the set of encountered frames.
frames map[cdp.FrameID]*cdp.Frame
@ -63,18 +61,22 @@ type TargetHandler struct {
}
// NewTargetHandler creates a new handler for the specified client target.
func NewTargetHandler(t client.Target, logf, debugf, errf func(string, ...interface{})) (*TargetHandler, error) {
conn, err := client.Dial(t.GetWebsocketURL())
func NewTargetHandler(urlstr string, opts ...TargetHandlerOption) (*TargetHandler, error) {
conn, err := Dial(urlstr)
if err != nil {
return nil, err
}
return &TargetHandler{
conn: conn,
logf: logf,
debugf: debugf,
errf: errf,
}, nil
h := &TargetHandler{
conn: conn,
errf: func(string, ...interface{}) {},
}
for _, o := range opts {
o(h)
}
return h, nil
}
// Run starts the processing of commands and events of the client target
@ -217,12 +219,11 @@ func (h *TargetHandler) read() (*cdproto.Message, error) {
return nil, err
}
h.debugf("-> %s", string(buf))
//h.debugf("-> %s", string(buf))
// unmarshal
msg := new(cdproto.Message)
err = json.Unmarshal(buf, msg)
if err != nil {
if err := json.Unmarshal(buf, msg); err != nil {
return nil, err
}
@ -332,7 +333,7 @@ func (h *TargetHandler) processCommand(cmd *cdproto.Message) error {
return err
}
h.debugf("<- %s", string(buf))
//h.debugf("<- %s", string(buf))
return h.conn.Write(buf)
}
@ -442,8 +443,6 @@ func (h *TargetHandler) GetRoot(ctxt context.Context) (*cdp.Node, error) {
// SetActive sets the currently active frame after a successful navigation.
func (h *TargetHandler) SetActive(ctxt context.Context, id cdp.FrameID) error {
var err error
// get frame
f, err := h.WaitFrame(ctxt, id)
if err != nil {
@ -461,7 +460,7 @@ func (h *TargetHandler) SetActive(ctxt context.Context, id cdp.FrameID) error {
// WaitFrame waits for a frame to be loaded using the provided context.
func (h *TargetHandler) WaitFrame(ctxt context.Context, id cdp.FrameID) (*cdp.Frame, error) {
// TODO: fix this
timeout := time.After(10 * time.Second)
timeout := time.After(time.Second)
for {
select {
@ -495,7 +494,7 @@ func (h *TargetHandler) WaitFrame(ctxt context.Context, id cdp.FrameID) (*cdp.Fr
// WaitNode waits for a node to be loaded using the provided context.
func (h *TargetHandler) WaitNode(ctxt context.Context, f *cdp.Frame, id cdp.NodeID) (*cdp.Node, error) {
// TODO: fix this
timeout := time.After(10 * time.Second)
timeout := time.After(time.Second)
for {
select {
@ -672,3 +671,5 @@ func (h *TargetHandler) domEvent(ctxt context.Context, ev interface{}) {
op(n)
}
type TargetHandlerOption func(*TargetHandler)

9
input.go
View File

@ -58,10 +58,8 @@ func MouseClickXY(x, y int64, opts ...MouseOption) Action {
// viewport.
func MouseClickNode(n *cdp.Node, opts ...MouseOption) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
var pos []int
err = EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, n.FullXPath()), &pos).Do(ctxt, h)
err := EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, n.FullXPath()), &pos).Do(ctxt, h)
if err != nil {
return err
}
@ -154,12 +152,9 @@ func ClickCount(n int) MouseOption {
// of well-known keys.
func KeyAction(keys string, opts ...KeyOption) Action {
return ActionFunc(func(ctxt context.Context, h cdp.Executor) error {
var err error
for _, r := range keys {
for _, k := range kb.Encode(r) {
err = k.Do(ctxt, h)
if err != nil {
if err := k.Do(ctxt, h); err != nil {
return err
}
}

98
input_test.go
View File

@ -23,13 +23,9 @@ const (
func TestMouseClickXY(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "input.html")
defer c.Release()
err = c.Run(defaultContext, Sleep(100*time.Millisecond))
if err != nil {
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
if err := Run(ctx, Sleep(100*time.Millisecond)); err != nil {
t.Fatal(err)
}
@ -43,18 +39,17 @@ func TestMouseClickXY(t *testing.T) {
}
for i, test := range tests {
err = c.Run(defaultContext, MouseClickXY(test.x, test.y))
if err != nil {
if err := Run(ctx, MouseClickXY(test.x, test.y)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
time.Sleep(50 * time.Millisecond)
var xstr, ystr string
err = c.Run(defaultContext, Value("#input1", &xstr, ByID))
if err != nil {
if err := Run(ctx, Value("#input1", &xstr, ByID)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
x, err := strconv.ParseInt(xstr, 10, 64)
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
@ -62,11 +57,10 @@ func TestMouseClickXY(t *testing.T) {
if x != test.x {
t.Fatalf("test %d expected x to be: %d, got: %d", i, test.x, x)
}
err = c.Run(defaultContext, Value("#input2", &ystr, ByID))
if err != nil {
if err := Run(ctx, Value("#input2", &ystr, ByID)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
y, err := strconv.ParseInt(ystr, 10, 64)
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
@ -97,31 +91,28 @@ func TestMouseClickNode(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "input.html")
defer c.Release()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
var err error
var nodes []*cdp.Node
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
err = c.Run(defaultContext, MouseClickNode(nodes[0], test.opt))
if err != nil {
if err := Run(ctx, MouseClickNode(nodes[0], test.opt)); err != nil {
t.Fatalf("got error: %v", err)
}
time.Sleep(50 * time.Millisecond)
var value string
err = c.Run(defaultContext, Value("#input3", &value, ByID))
if err != nil {
if err := Run(ctx, Value("#input3", &value, ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}
@ -146,42 +137,41 @@ func TestMouseClickOffscreenNode(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "input.html")
defer c.Release()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
var err error
var nodes []*cdp.Node
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
var ok bool
err = c.Run(defaultContext, EvaluateAsDevTools(fmt.Sprintf(inViewportJS, nodes[0].FullXPath()), &ok))
if err != nil {
if err := Run(ctx, EvaluateAsDevTools(fmt.Sprintf(inViewportJS, nodes[0].FullXPath()), &ok)); err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected node to be offscreen")
}
for i := test.exp; i > 0; i-- {
err = c.Run(defaultContext, MouseClickNode(nodes[0]))
if err != nil {
if err := Run(ctx, MouseClickNode(nodes[0])); err != nil {
t.Fatalf("got error: %v", err)
}
}
time.Sleep(100 * time.Millisecond)
var value int
err = c.Run(defaultContext, Evaluate("window.document.test_i", &value))
if err != nil {
if err := Run(ctx, Evaluate("window.document.test_i", &value)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %d, got: %d", test.exp, value)
}
@ -208,34 +198,29 @@ func TestKeyAction(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "input.html")
defer c.Release()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
var err error
var nodes []*cdp.Node
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
err = c.Run(defaultContext, Focus(test.sel, test.by))
if err != nil {
if err := Run(ctx, Focus(test.sel, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, KeyAction(test.exp))
if err != nil {
if err := Run(ctx, KeyAction(test.exp)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}
@ -262,29 +247,26 @@ func TestKeyActionNode(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "input.html")
defer c.Release()
ctx, cancel := testAllocate(t, "input.html")
defer cancel()
var err error
var nodes []*cdp.Node
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodes) != 1 {
t.Fatalf("expected nodes to have exactly 1 element, got: %d", len(nodes))
}
err = c.Run(defaultContext, KeyActionNode(nodes[0], test.exp))
if err != nil {
if err := Run(ctx, KeyActionNode(nodes[0], test.exp)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Fatalf("expected to have value %s, got: %s", test.exp, value)
}

22
kb/gen.go
View File

@ -72,8 +72,6 @@ func main() {
}
func run() error {
var err error
// special characters
keys := map[rune]kb.Key{
'\b': {"Backspace", "Backspace", "", "", int64('\b'), int64('\b'), false, false},
@ -82,8 +80,7 @@ func run() error {
}
// load keys
err = loadKeys(keys)
if err != nil {
if err := loadKeys(keys); err != nil {
return err
}
@ -94,24 +91,19 @@ func run() error {
}
// output
err = ioutil.WriteFile(
*flagOut,
if err := ioutil.WriteFile(*flagOut,
[]byte(fmt.Sprintf(hdr, *flagPkg, string(constBuf), string(mapBuf))),
0644,
)
if err != nil {
0644); err != nil {
return err
}
// format
err = exec.Command("goimports", "-w", *flagOut).Run()
if err != nil {
if err := exec.Command("goimports", "-w", *flagOut).Run(); err != nil {
return err
}
// format
err = exec.Command("gofmt", "-s", "-w", *flagOut).Run()
if err != nil {
if err := exec.Command("gofmt", "-s", "-w", *flagOut).Run(); err != nil {
return err
}
@ -120,8 +112,6 @@ func run() error {
// loadKeys loads the dom key definitions from the chromium source tree.
func loadKeys(keys map[rune]kb.Key) error {
var err error
// load key converter data
keycodeConverterMap, err := loadKeycodeConverterData()
if err != nil {
@ -444,8 +434,6 @@ var defineRE = regexp.MustCompile(`(?m)^#define\s+(.+?)\s+([0-9A-Fx]+)`)
// loadPosixWinKeyboardCodes loads the native and windows keyboard scan codes
// mapped to the DOM key.
func loadPosixWinKeyboardCodes() (map[string][]int64, error) {
var err error
lookup := map[string]string{
// mac alias
"VKEY_LWIN": "0x5B",

227
nav_test.go
View File

@ -11,37 +11,31 @@ import (
func TestNavigate(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
expurl, exptitle := testdataDir+"/image.html", "this is title"
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
if err := Run(ctx, Navigate(expurl)); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, WaitVisible(`#icon-brankas`, ByID))
if err != nil {
if err := Run(ctx, WaitVisible(`#icon-brankas`, ByID)); err != nil {
t.Fatal(err)
}
var urlstr string
err = c.Run(defaultContext, Location(&urlstr))
if err != nil {
if err := Run(ctx, Location(&urlstr)); err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(urlstr, expurl) {
t.Errorf("expected to be on image.html, at: %s", urlstr)
}
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to contain google, instead title is: %s", title)
}
@ -50,10 +44,9 @@ func TestNavigate(t *testing.T) {
func TestNavigationEntries(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
time.Sleep(50 * time.Millisecond)
tests := []string{
"form.html",
@ -62,38 +55,33 @@ func TestNavigationEntries(t *testing.T) {
var entries []*page.NavigationEntry
var index int64
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
if err := Run(ctx, NavigationEntries(&index, &entries)); err != nil {
t.Fatal(err)
}
if len(entries) != 1 {
t.Errorf("expected to have 1 navigation entry: got %d", len(entries))
if len(entries) != 2 {
t.Errorf("expected to have 2 navigation entry: got %d", len(entries))
}
if index != 0 {
t.Errorf("expected navigation index is 0, got: %d", index)
if index != 1 {
t.Errorf("expected navigation index is 1, got: %d", index)
}
expIdx, expEntries := 1, 2
expIdx, expEntries := 2, 3
for i, url := range tests {
err = c.Run(defaultContext, Navigate(testdataDir+"/"+url))
if err != nil {
if err := Run(ctx, Navigate(testdataDir+"/"+url)); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
if err := Run(ctx, NavigationEntries(&index, &entries)); err != nil {
t.Fatal(err)
}
if len(entries) != expEntries {
t.Errorf("test %d expected to have %d navigation entry: got %d", i, expEntries, len(entries))
}
if index != int64(i+1) {
t.Errorf("test %d expected navigation index is %d, got: %d", i, i, index)
if want := int64(i + 2); index != want {
t.Errorf("test %d expected navigation index is %d, got: %d", i, want, index)
}
expIdx++
@ -104,44 +92,36 @@ func TestNavigationEntries(t *testing.T) {
func TestNavigateToHistoryEntry(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
var entries []*page.NavigationEntry
var index int64
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
if err := Run(ctx, Navigate(testdataDir+"/image.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigationEntries(&index, &entries))
if err != nil {
if err := Run(ctx, NavigationEntries(&index, &entries)); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateToHistoryEntry(entries[index].ID))
if err != nil {
if err := Run(ctx, NavigateToHistoryEntry(entries[index].ID)); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}
if title != entries[index].Title {
t.Errorf("expected title to be %s, instead title is: %s", entries[index].Title, title)
}
@ -150,43 +130,35 @@ func TestNavigateToHistoryEntry(t *testing.T) {
func TestNavigateBack(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
if err := Run(ctx, Title(&exptitle)); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
if err := Run(ctx, Navigate(testdataDir+"/image.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateBack())
if err != nil {
if err := Run(ctx, NavigateBack()); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -195,50 +167,39 @@ func TestNavigateBack(t *testing.T) {
func TestNavigateForward(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
if err := Run(ctx, Navigate(testdataDir+"/image.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
if err := Run(ctx, Title(&exptitle)); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, NavigateBack())
if err != nil {
if err := Run(ctx, NavigateBack()); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, NavigateForward())
if err != nil {
if err := Run(ctx, NavigateForward()); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -247,55 +208,44 @@ func TestNavigateForward(t *testing.T) {
func TestStop(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
t.Fatal(err)
}
if err := Run(ctx, Stop()); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Stop())
if err != nil {
t.Fatal(err)
}
}
func TestReload(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var exptitle string
err = c.Run(defaultContext, Title(&exptitle))
if err != nil {
if err := Run(ctx, Title(&exptitle)); err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Reload())
if err != nil {
if err := Run(ctx, Reload()); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}
if title != exptitle {
t.Errorf("expected title to be %s, instead title is: %s", exptitle, title)
}
@ -304,21 +254,16 @@ func TestReload(t *testing.T) {
func TestCaptureScreenshot(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(testdataDir+"/image.html"))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(testdataDir+"/image.html")); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var buf []byte
err = c.Run(defaultContext, CaptureScreenshot(&buf))
if err != nil {
if err := Run(ctx, CaptureScreenshot(&buf)); err != nil {
t.Fatal(err)
}
@ -331,18 +276,16 @@ func TestCaptureScreenshot(t *testing.T) {
/*func TestAddOnLoadScript(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
var scriptID page.ScriptIdentifier
err = c.Run(defaultContext, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
err = Run(ctx, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
err = Run(ctx, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
@ -358,13 +301,11 @@ func TestCaptureScreenshot(t *testing.T) {
func TestRemoveOnLoadScript(t *testing.T) {
t.Parallel()
var err error
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
var scriptID page.ScriptIdentifier
err = c.Run(defaultContext, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
err = Run(ctx, AddOnLoadScript(`window.alert("TEST")`, &scriptID))
if err != nil {
t.Fatal(err)
}
@ -373,12 +314,12 @@ func TestRemoveOnLoadScript(t *testing.T) {
t.Fatal("got empty script ID")
}
err = c.Run(defaultContext, RemoveOnLoadScript(scriptID))
err = Run(ctx, RemoveOnLoadScript(scriptID))
if err != nil {
t.Fatal(err)
}
err = c.Run(defaultContext, Navigate(testdataDir+"/form.html"))
err = Run(ctx, Navigate(testdataDir+"/form.html"))
if err != nil {
t.Fatal(err)
}
@ -389,22 +330,18 @@ func TestRemoveOnLoadScript(t *testing.T) {
func TestLocation(t *testing.T) {
t.Parallel()
var err error
expurl := testdataDir + "/form.html"
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(expurl)); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var urlstr string
err = c.Run(defaultContext, Location(&urlstr))
if err != nil {
if err := Run(ctx, Location(&urlstr)); err != nil {
t.Fatal(err)
}
@ -416,22 +353,18 @@ func TestLocation(t *testing.T) {
func TestTitle(t *testing.T) {
t.Parallel()
var err error
expurl, exptitle := testdataDir+"/image.html", "this is title"
c := testAllocate(t, "")
defer c.Release()
err = c.Run(defaultContext, Navigate(expurl))
if err != nil {
ctx, cancel := testAllocate(t, "")
defer cancel()
if err := Run(ctx, Navigate(expurl)); err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond)
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatal(err)
}

219
pool.go
View File

@ -1,219 +0,0 @@
package chromedp
import (
"context"
"fmt"
"log"
"net"
"sync"
"github.com/chromedp/chromedp/runner"
)
// Pool manages a pool of running Chrome processes.
type Pool struct {
// start is the start port.
start int
// end is the end port.
end int
// res are the running chrome resources.
res map[int]*Res
// logging funcs
logf, debugf, errf func(string, ...interface{})
rw sync.RWMutex
}
// NewPool creates a new Chrome runner pool.
func NewPool(opts ...PoolOption) (*Pool, error) {
p := &Pool{
start: DefaultPoolStartPort,
end: DefaultPoolEndPort,
res: make(map[int]*Res),
logf: log.Printf,
debugf: func(string, ...interface{}) {},
}
// apply opts
for _, o := range opts {
if err := o(p); err != nil {
return nil, err
}
}
if p.errf == nil {
p.errf = func(s string, v ...interface{}) {
p.logf("ERROR: "+s, v...)
}
}
return p, nil
}
// Shutdown releases all the pool resources.
func (p *Pool) Shutdown() error {
p.rw.Lock()
defer p.rw.Unlock()
for _, r := range p.res {
r.cancel()
}
return nil
}
// Allocate creates a new process runner and returns it.
func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption) (*Res, error) {
var err error
r := p.next(ctxt)
// Check if the port is available first. If it's not, Chrome will print
// an "address already in use" error, but it will otherwise keep
// running. This can lead to Allocate succeeding, while the chrome
// process isn't actually listening on the port we need.
l, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", r.port))
if err != nil {
// we can't use this port, e.g. address already in use
p.errf("pool could not allocate runner on port %d: %v", r.port, err)
return nil, err
}
l.Close()
p.debugf("pool allocating %d", r.port)
// create runner
r.r, err = runner.New(append([]runner.CommandLineOption{
runner.ExecPath(runner.LookChromeNames("headless_shell")),
runner.RemoteDebuggingPort(r.port),
runner.NoDefaultBrowserCheck,
runner.NoFirstRun,
runner.Headless,
}, opts...)...)
if err != nil {
defer r.Release()
p.errf("pool could not allocate runner on port %d: %v", r.port, err)
return nil, err
}
// start runner
err = r.r.Start(r.ctxt)
if err != nil {
defer r.Release()
p.errf("pool could not start runner on port %d: %v", r.port, err)
return nil, err
}
// setup cdp
r.c, err = New(
r.ctxt, WithRunner(r.r),
WithLogf(p.logf), WithDebugf(p.debugf), WithErrorf(p.errf),
)
if err != nil {
defer r.Release()
p.errf("pool could not connect to %d: %v", r.port, err)
return nil, err
}
return r, nil
}
// next returns the next available res.
func (p *Pool) next(ctxt context.Context) *Res {
p.rw.Lock()
defer p.rw.Unlock()
var found bool
var i int
for i = p.start; i < p.end; i++ {
if _, ok := p.res[i]; !ok {
found = true
break
}
}
if !found {
panic("no ports available")
}
r := &Res{
p: p,
port: i,
}
r.ctxt, r.cancel = context.WithCancel(ctxt)
p.res[i] = r
return r
}
// Res is a pool resource.
type Res struct {
p *Pool
ctxt context.Context
cancel func()
port int
r *runner.Runner
c *CDP
}
// Release releases the pool resource.
func (r *Res) Release() error {
r.cancel()
var err error
if r.c != nil {
err = r.c.Wait()
}
defer r.p.debugf("pool released %d", r.port)
r.p.rw.Lock()
defer r.p.rw.Unlock()
delete(r.p.res, r.port)
return err
}
// Port returns the allocated port for the pool resource.
func (r *Res) Port() int {
return r.port
}
// URL returns a formatted URL for the pool resource.
func (r *Res) URL() string {
return fmt.Sprintf("http://localhost:%d/json", r.port)
}
// CDP returns the actual CDP instance.
func (r *Res) CDP() *CDP {
return r.c
}
// Run runs an action.
func (r *Res) Run(ctxt context.Context, a Action) error {
return r.c.Run(ctxt, a)
}
// PoolOption is a pool option.
type PoolOption func(*Pool) error
// PortRange is a pool option to set the port range to use.
func PortRange(start, end int) PoolOption {
return func(p *Pool) error {
p.start = start
p.end = end
return nil
}
}
// PoolLog is a pool option to set the logging to use for the pool.
func PoolLog(logf, debugf, errf func(string, ...interface{})) PoolOption {
return func(p *Pool) error {
p.logf, p.debugf, p.errf = logf, debugf, errf
return nil
}
}

47
pool_test.go
View File

@ -1,47 +0,0 @@
package chromedp
import (
"context"
"net"
"strconv"
"strings"
"testing"
)
func TestAllocatePortInUse(t *testing.T) {
t.Parallel()
// take a random available port
l, err := net.Listen("tcp4", "localhost:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
ctxt, cancel := context.WithCancel(context.Background())
defer cancel()
// make the pool use the port already in use via a port range
_, portStr, _ := net.SplitHostPort(l.Addr().String())
port, _ := strconv.Atoi(portStr)
pool, err := NewPool(
PortRange(port, port+1),
// skip the error log from the used port
PoolLog(nil, nil, func(string, ...interface{}) {}),
)
if err != nil {
t.Fatal(err)
}
c, err := pool.Allocate(ctxt)
if err != nil {
want := "address already in use"
got := err.Error()
if !strings.Contains(got, want) {
t.Fatalf("wanted error to contain %q, but got %q", want, got)
}
} else {
t.Fatal("wanted Allocate to error if port is in use")
c.Release()
}
}

3
query.go
View File

@ -484,8 +484,7 @@ func Screenshot(sel interface{}, picbuf *[]byte, opts ...QueryOption) Action {
// encode
var croppedBuf bytes.Buffer
err = png.Encode(&croppedBuf, cropped)
if err != nil {
if err := png.Encode(&croppedBuf, cropped); err != nil {
return err
}

281
query_test.go
View File

@ -20,8 +20,8 @@ import (
func TestNodes(t *testing.T) {
t.Parallel()
c := testAllocate(t, "table.html")
defer c.Release()
ctx, cancel := testAllocate(t, "table.html")
defer cancel()
tests := []struct {
sel string
@ -34,13 +34,12 @@ func TestNodes(t *testing.T) {
{"#footer", ByID, 1},
}
var err error
for i, test := range tests {
var nodes []*cdp.Node
err = c.Run(defaultContext, Nodes(test.sel, &nodes, test.by))
if err != nil {
if err := Run(ctx, Nodes(test.sel, &nodes, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if len(nodes) != test.len {
t.Errorf("test %d expected to have %d nodes: got %d", i, test.len, len(nodes))
}
@ -50,8 +49,8 @@ func TestNodes(t *testing.T) {
func TestNodeIDs(t *testing.T) {
t.Parallel()
c := testAllocate(t, "table.html")
defer c.Release()
ctx, cancel := testAllocate(t, "table.html")
defer cancel()
tests := []struct {
sel string
@ -64,13 +63,12 @@ func TestNodeIDs(t *testing.T) {
{"#footer", ByID, 1},
}
var err error
for i, test := range tests {
var ids []cdp.NodeID
err = c.Run(defaultContext, NodeIDs(test.sel, &ids, test.by))
if err != nil {
if err := Run(ctx, NodeIDs(test.sel, &ids, test.by)); err != nil {
t.Fatal(err)
}
if len(ids) != test.len {
t.Errorf("test %d expected to have %d node id's: got %d", i, test.len, len(ids))
}
@ -80,8 +78,8 @@ func TestNodeIDs(t *testing.T) {
func TestFocusBlur(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
tests := []struct {
sel string
@ -93,35 +91,31 @@ func TestFocusBlur(t *testing.T) {
{"#input1", ByID},
}
err := c.Run(defaultContext, Click("#input1", ByID))
err := Run(ctx, Click("#input1", ByID))
if err != nil {
t.Fatal(err)
}
for i, test := range tests {
err = c.Run(defaultContext, Focus(test.sel, test.by))
if err != nil {
if err := Run(ctx, Focus(test.sel, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if value != "9999" {
t.Errorf("test %d expected value is '9999', got: '%s'", i, value)
}
err = c.Run(defaultContext, Blur(test.sel, test.by))
if err != nil {
if err := Run(ctx, Blur(test.sel, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if value != "0" {
t.Errorf("test %d expected value is '0', got: '%s'", i, value)
}
@ -131,8 +125,8 @@ func TestFocusBlur(t *testing.T) {
func TestDimensions(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -146,13 +140,12 @@ func TestDimensions(t *testing.T) {
{"#icon-github", ByID, 120, 120},
}
var err error
for i, test := range tests {
var model *dom.BoxModel
err = c.Run(defaultContext, Dimensions(test.sel, &model))
if err != nil {
if err := Run(ctx, Dimensions(test.sel, &model)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if model.Height != test.height || model.Width != test.width {
t.Errorf("test %d expected %dx%d, got: %dx%d", i, test.width, test.height, model.Height, model.Width)
}
@ -162,8 +155,8 @@ func TestDimensions(t *testing.T) {
func TestText(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
tests := []struct {
sel string
@ -176,13 +169,12 @@ func TestText(t *testing.T) {
{"/html/body/form/span[2]", BySearch, "keyword"},
}
var err error
for i, test := range tests {
var text string
err = c.Run(defaultContext, Text(test.sel, &text, test.by))
if err != nil {
if err := Run(ctx, Text(test.sel, &text, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if text != test.exp {
t.Errorf("test %d expected `%s`, got: %s", i, test.exp, text)
}
@ -217,27 +209,24 @@ func TestClear(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
var val string
err := c.Run(defaultContext, Value(test.sel, &val, test.by))
err := Run(ctx, Value(test.sel, &val, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if val == "" {
t.Errorf("expected `%s` to have non empty value", test.sel)
}
err = c.Run(defaultContext, Clear(test.sel, test.by))
if err != nil {
if err := Run(ctx, Clear(test.sel, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, Value(test.sel, &val, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, Value(test.sel, &val, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
if val != "" {
t.Errorf("expected empty value for `%s`, got: %s", test.sel, val)
}
@ -264,24 +253,22 @@ func TestReset(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
err := c.Run(defaultContext, SetValue(test.sel, test.value, test.by))
err := Run(ctx, SetValue(test.sel, test.value, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, Reset(test.sel, test.by))
if err != nil {
if err := Run(ctx, Reset(test.sel, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != test.exp {
t.Errorf("expected value after reset is %s, got: '%s'", test.exp, value)
}
@ -292,8 +279,8 @@ func TestReset(t *testing.T) {
func TestValue(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
tests := []struct {
sel string
@ -305,13 +292,12 @@ func TestValue(t *testing.T) {
{`#keyword`, ByID},
}
var err error
for i, test := range tests {
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if value != "chromedp" {
t.Errorf("test %d expected `chromedp`, got: %s", i, value)
}
@ -335,19 +321,19 @@ func TestSetValue(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
err := c.Run(defaultContext, SetValue(test.sel, "FOOBAR", test.by))
err := Run(ctx, SetValue(test.sel, "FOOBAR", test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(test.sel, &value, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &value, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != "FOOBAR" {
t.Errorf("expected `FOOBAR`, got: %s", value)
}
@ -358,8 +344,8 @@ func TestSetValue(t *testing.T) {
func TestAttributes(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -392,11 +378,9 @@ func TestAttributes(t *testing.T) {
}},
}
var err error
for i, test := range tests {
var attrs map[string]string
err = c.Run(defaultContext, Attributes(test.sel, &attrs, test.by))
if err != nil {
if err := Run(ctx, Attributes(test.sel, &attrs, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
@ -409,8 +393,8 @@ func TestAttributes(t *testing.T) {
func TestAttributesAll(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -433,11 +417,9 @@ func TestAttributesAll(t *testing.T) {
},
}
var err error
for i, test := range tests {
var attrs []map[string]string
err = c.Run(defaultContext, AttributesAll(test.sel, &attrs, test.by))
if err != nil {
if err := Run(ctx, AttributesAll(test.sel, &attrs, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
@ -495,17 +477,16 @@ func TestSetAttributes(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
err := c.Run(defaultContext, SetAttributes(test.sel, test.attrs, test.by))
err := Run(ctx, SetAttributes(test.sel, test.attrs, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
var attrs map[string]string
err = c.Run(defaultContext, Attributes(test.sel, &attrs, test.by))
if err != nil {
if err := Run(ctx, Attributes(test.sel, &attrs, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
@ -519,8 +500,8 @@ func TestSetAttributes(t *testing.T) {
func TestAttributeValue(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -534,13 +515,10 @@ func TestAttributeValue(t *testing.T) {
{"#icon-github", ByID, "alt", "How people build software"},
}
var err error
for i, test := range tests {
var value string
var ok bool
err = c.Run(defaultContext, AttributeValue(test.sel, test.attr, &value, &ok, test.by))
if err != nil {
if err := Run(ctx, AttributeValue(test.sel, test.attr, &value, &ok, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
@ -573,20 +551,20 @@ func TestSetAttributeValue(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
err := c.Run(defaultContext, SetAttributeValue(test.sel, test.attr, test.exp, test.by))
err := Run(ctx, SetAttributeValue(test.sel, test.attr, test.exp, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
var ok bool
err = c.Run(defaultContext, AttributeValue(test.sel, test.attr, &value, &ok, test.by))
if err != nil {
if err := Run(ctx, AttributeValue(test.sel, test.attr, &value, &ok, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if !ok {
t.Fatalf("failed to get attribute %s on %s", test.attr, test.sel)
}
@ -616,20 +594,20 @@ func TestRemoveAttribute(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
err := c.Run(defaultContext, RemoveAttribute(test.sel, test.attr))
err := Run(ctx, RemoveAttribute(test.sel, test.attr))
if err != nil {
t.Fatalf("got error: %v", err)
}
var value string
var ok bool
err = c.Run(defaultContext, AttributeValue(test.sel, test.attr, &value, &ok, test.by))
if err != nil {
if err := Run(ctx, AttributeValue(test.sel, test.attr, &value, &ok, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if ok || value != "" {
t.Fatalf("expected attribute %s removed from element %s", test.attr, test.sel)
}
@ -654,24 +632,22 @@ func TestClick(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
err := c.Run(defaultContext, Click(test.sel, test.by))
err := Run(ctx, Click(test.sel, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitVisible("#icon-brankas", ByID))
if err != nil {
if err := Run(ctx, WaitVisible("#icon-brankas", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatalf("got error: %v", err)
}
if title != "this is title" {
t.Errorf("expected title to be 'chromedp - Google Search', got: '%s'", title)
}
@ -696,10 +672,10 @@ func TestDoubleClick(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
err := c.Run(defaultContext, DoubleClick(test.sel, test.by))
err := Run(ctx, DoubleClick(test.sel, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
@ -707,10 +683,10 @@ func TestDoubleClick(t *testing.T) {
time.Sleep(50 * time.Millisecond)
var value string
err = c.Run(defaultContext, Value("#input1", &value, ByID))
if err != nil {
if err := Run(ctx, Value("#input1", &value, ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != "1" {
t.Errorf("expected value to be '1', got: '%s'", value)
}
@ -739,19 +715,19 @@ func TestSendKeys(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "visible.html")
defer c.Release()
ctx, cancel := testAllocate(t, "visible.html")
defer cancel()
err := c.Run(defaultContext, SendKeys(test.sel, test.keys, test.by))
err := Run(ctx, SendKeys(test.sel, test.keys, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
var val string
err = c.Run(defaultContext, Value(test.sel, &val, test.by))
if err != nil {
if err := Run(ctx, Value(test.sel, &val, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
if val != test.exp {
t.Errorf("expected value %s, got: %s", test.exp, val)
}
@ -762,8 +738,8 @@ func TestSendKeys(t *testing.T) {
func TestScreenshot(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -775,11 +751,9 @@ func TestScreenshot(t *testing.T) {
{"#icon-github", ByID},
}
var err error
for i, test := range tests {
var buf []byte
err = c.Run(defaultContext, Screenshot(test.sel, &buf))
if err != nil {
if err := Run(ctx, Screenshot(test.sel, &buf)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
@ -807,24 +781,22 @@ func TestSubmit(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "form.html")
defer c.Release()
ctx, cancel := testAllocate(t, "form.html")
defer cancel()
err := c.Run(defaultContext, Submit(test.sel, test.by))
err := Run(ctx, Submit(test.sel, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitVisible("#icon-brankas", ByID))
if err != nil {
if err := Run(ctx, WaitVisible("#icon-brankas", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var title string
err = c.Run(defaultContext, Title(&title))
if err != nil {
if err := Run(ctx, Title(&title)); err != nil {
t.Fatalf("got error: %v", err)
}
if title != "this is title" {
t.Errorf("expected title to be 'this is title', got: '%s'", title)
}
@ -849,13 +821,13 @@ func TestComputedStyle(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
time.Sleep(50 * time.Millisecond)
var styles []*css.ComputedProperty
err := c.Run(defaultContext, ComputedStyle(test.sel, &styles, test.by))
err := Run(ctx, ComputedStyle(test.sel, &styles, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
@ -867,16 +839,12 @@ func TestComputedStyle(t *testing.T) {
}
}
}
err = c.Run(defaultContext, Click("#input1", ByID))
if err != nil {
if err := Run(ctx, Click("#input1", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
time.Sleep(50 * time.Millisecond)
err = c.Run(defaultContext, ComputedStyle(test.sel, &styles, test.by))
if err != nil {
if err := Run(ctx, ComputedStyle(test.sel, &styles, test.by)); err != nil {
t.Fatalf("got error: %v", err)
}
@ -908,13 +876,13 @@ func TestMatchedStyle(t *testing.T) {
t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
time.Sleep(50 * time.Millisecond)
var styles *css.GetMatchedStylesForNodeReturns
err := c.Run(defaultContext, MatchedStyle(test.sel, &styles, test.by))
err := Run(ctx, MatchedStyle(test.sel, &styles, test.by))
if err != nil {
t.Fatalf("got error: %v", err)
}
@ -957,10 +925,10 @@ func TestFileUpload(t *testing.T) {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
if _, err = tmpfile.WriteString(uploadHTML); err != nil {
if _, err := tmpfile.WriteString(uploadHTML); err != nil {
t.Fatal(err)
}
if err = tmpfile.Close(); err != nil {
if err := tmpfile.Close(); err != nil {
t.Fatal(err)
}
@ -977,19 +945,19 @@ func TestFileUpload(t *testing.T) {
// parallel
//t.Parallel()
c := testAllocate(t, "")
defer c.Release()
ctx, cancel := testAllocate(t, "")
defer cancel()
var result string
err = c.Run(defaultContext, Tasks{
if err := Run(ctx, Tasks{
Navigate(s.URL),
test.a,
Click(`input[name="submit"]`),
Text(`#result`, &result, ByID, NodeVisible),
})
if err != nil {
}); err != nil {
t.Fatalf("test %d expected no error, got: %v", i, err)
}
if result != fmt.Sprintf("%d", len(uploadHTML)) {
t.Errorf("test %d expected result to be %d, got: %s", i, len(uploadHTML), result)
}
@ -1000,8 +968,8 @@ func TestFileUpload(t *testing.T) {
func TestInnerHTML(t *testing.T) {
t.Parallel()
c := testAllocate(t, "table.html")
defer c.Release()
ctx, cancel := testAllocate(t, "table.html")
defer cancel()
tests := []struct {
sel string
@ -1011,13 +979,12 @@ func TestInnerHTML(t *testing.T) {
{"thead", ByQueryAll},
{"thead", ByQuery},
}
var err error
for i, test := range tests {
var html string
err = c.Run(defaultContext, InnerHTML(test.sel, &html))
if err != nil {
if err := Run(ctx, InnerHTML(test.sel, &html)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if html == "" {
t.Fatalf("test %d: InnerHTML is empty", i)
}
@ -1027,8 +994,8 @@ func TestInnerHTML(t *testing.T) {
func TestOuterHTML(t *testing.T) {
t.Parallel()
c := testAllocate(t, "table.html")
defer c.Release()
ctx, cancel := testAllocate(t, "table.html")
defer cancel()
tests := []struct {
sel string
@ -1038,13 +1005,12 @@ func TestOuterHTML(t *testing.T) {
{"thead tr", ByQueryAll},
{"thead tr", ByQuery},
}
var err error
for i, test := range tests {
var html string
err = c.Run(defaultContext, OuterHTML(test.sel, &html))
if err != nil {
if err := Run(ctx, OuterHTML(test.sel, &html)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
if html == "" {
t.Fatalf("test %d: OuterHTML is empty", i)
}
@ -1054,8 +1020,8 @@ func TestOuterHTML(t *testing.T) {
func TestScrollIntoView(t *testing.T) {
t.Parallel()
c := testAllocate(t, "image.html")
defer c.Release()
ctx, cancel := testAllocate(t, "image.html")
defer cancel()
tests := []struct {
sel string
@ -1066,12 +1032,11 @@ func TestScrollIntoView(t *testing.T) {
{"img", ByQuery},
{"#icon-github", ByID},
}
var err error
for i, test := range tests {
err = c.Run(defaultContext, ScrollIntoView(test.sel, test.by))
if err != nil {
if err := Run(ctx, ScrollIntoView(test.sel, test.by)); err != nil {
t.Fatalf("test %d got error: %v", i, err)
}
// TODO test scroll event
}
}

13
runner/path_darwin.go
View File

@ -1,13 +0,0 @@
// +build darwin
package runner
const (
// DefaultChromePath is the default path to use for Chrome if the
// executable is not in $PATH.
DefaultChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`
)
// DefaultChromeNames are the default Chrome executable names to look for in
// $PATH.
var DefaultChromeNames []string

19
runner/path_unix.go
View File

@ -1,19 +0,0 @@
// +build linux freebsd netbsd openbsd
package runner
const (
// DefaultChromePath is the default path to use for Chrome if the
// executable is not in $PATH.
DefaultChromePath = "/usr/bin/google-chrome"
)
// DefaultChromeNames are the default Chrome executable names to look for in
// $PATH.
var DefaultChromeNames = []string{
"google-chrome",
"chromium-browser",
"chromium",
"google-chrome-beta",
"google-chrome-unstable",
}

13
runner/path_windows.go
View File

@ -1,13 +0,0 @@
// +build windows
package runner
const (
// DefaultChromePath is the default path to use for Chrome if the
// executable is not in %PATH%.
DefaultChromePath = `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`
)
// DefaultChromeNames are the default Chrome executable names to look for in
// %PATH%.
var DefaultChromeNames = []string{`chrome.exe`}

486
runner/runner.go
View File

@ -1,486 +0,0 @@
// Package runner provides a Chrome process runner.
package runner
import (
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"regexp"
"runtime"
"sync"
"syscall"
"github.com/chromedp/chromedp/client"
)
const (
// DefaultUserDataDirPrefix is the default user data directory prefix.
DefaultUserDataDirPrefix = "chromedp-runner.%d."
)
// Error is a runner error.
type Error string
// Error satisfies the error interface.
func (err Error) Error() string {
return string(err)
}
// Error values.
const (
// ErrAlreadyStarted is the already started error.
ErrAlreadyStarted Error = "already started"
// ErrAlreadyWaiting is the already waiting error.
ErrAlreadyWaiting Error = "already waiting"
// ErrInvalidURLs is the invalid url-opts error.
ErrInvalidURLOpts Error = "invalid url-opts"
// ErrInvalidCmdOpts is the invalid cmd-opts error.
ErrInvalidCmdOpts Error = "invalid cmd-opts"
// ErrInvalidProcessOpts is the invalid process-opts error.
ErrInvalidProcessOpts Error = "invalid process-opts"
// ErrInvalidExecPath is the invalid exec-path error.
ErrInvalidExecPath Error = "invalid exec-path"
)
// Runner holds information about a running Chrome process.
type Runner struct {
opts map[string]interface{}
cmd *exec.Cmd
waiting bool
rw sync.RWMutex
}
// New creates a new Chrome process using the supplied command line options.
func New(opts ...CommandLineOption) (*Runner, error) {
var err error
cliOpts := make(map[string]interface{})
// apply opts
for _, o := range opts {
if err = o(cliOpts); err != nil {
return nil, err
}
}
// set default Chrome options if exec-path not provided
if _, ok := cliOpts["exec-path"]; !ok {
cliOpts["exec-path"] = LookChromeNames()
for k, v := range map[string]interface{}{
"no-first-run": true,
"no-default-browser-check": true,
"remote-debugging-port": 9222,
} {
if _, ok := cliOpts[k]; !ok {
cliOpts[k] = v
}
}
}
// add KillProcessGroup and ForceKill if no other cmd opts provided
if _, ok := cliOpts["cmd-opts"]; !ok {
for _, o := range []CommandLineOption{KillProcessGroup, ForceKill} {
if err = o(cliOpts); err != nil {
return nil, err
}
}
}
return &Runner{
opts: cliOpts,
}, nil
}
// cliOptRE is a regular expression to validate a chrome cli option.
var cliOptRE = regexp.MustCompile(`^[a-z0-9\-]+$`)
// buildOpts generates the command line options for Chrome.
func (r *Runner) buildOpts() []string {
var opts []string
var urls []string
// process opts
for k, v := range r.opts {
if !cliOptRE.MatchString(k) || v == nil {
continue
}
switch k {
case "exec-path", "cmd-opts", "process-opts":
continue
case "url-opts":
urls = v.([]string)
default:
switch z := v.(type) {
case bool:
if z {
opts = append(opts, "--"+k)
}
case string:
opts = append(opts, "--"+k+"="+z)
default:
opts = append(opts, "--"+k+"="+fmt.Sprintf("%v", v))
}
}
}
if urls == nil {
urls = append(urls, "about:blank")
}
return append(opts, urls...)
}
// Start starts a Chrome process using the specified context. The Chrome
// process can be terminated by closing the passed context.
func (r *Runner) Start(ctxt context.Context, opts ...string) error {
var err error
var ok bool
r.rw.RLock()
cmd := r.cmd
r.rw.RUnlock()
if cmd != nil {
return ErrAlreadyStarted
}
// set user data dir, if not provided
_, ok = r.opts["user-data-dir"]
if !ok {
r.opts["user-data-dir"], err = ioutil.TempDir(
defaultUserDataTmpDir, fmt.Sprintf(DefaultUserDataDirPrefix, r.Port()),
)
if err != nil {
return err
}
}
// get exec path
var execPath string
if p, ok := r.opts["exec-path"]; ok {
execPath, ok = p.(string)
if !ok {
return ErrInvalidExecPath
}
}
// ensure execPath is valid
if execPath == "" {
return ErrInvalidExecPath
}
// create cmd
r.cmd = exec.CommandContext(ctxt, execPath, append(r.buildOpts(), opts...)...)
// apply cmd opts
if cmdOpts, ok := r.opts["cmd-opts"]; ok {
for _, co := range cmdOpts.([]func(*exec.Cmd) error) {
if err = co(r.cmd); err != nil {
return err
}
}
}
// start process
if err = r.cmd.Start(); err != nil {
return err
}
// apply process opts
if processOpts, ok := r.opts["process-opts"]; ok {
for _, po := range processOpts.([]func(*os.Process) error) {
if err = po(r.cmd.Process); err != nil {
// TODO: do something better here, as we want to kill
// the child process, do cleanup, etc.
panic(err)
//return err
}
}
}
return nil
}
// Shutdown shuts down the Chrome process. Currently only has support for
// SIGTERM in darwin and linux systems
func (r *Runner) Shutdown(ctxt context.Context, opts ...client.Option) error {
var err error
cl := r.Client(opts...)
targets, err := cl.ListPageTargets(ctxt)
if err != nil {
return err
}
var wg sync.WaitGroup
errs := make([]error, len(targets))
for i, t := range targets {
wg.Add(1)
go func(wg *sync.WaitGroup, i int, t client.Target) {
defer wg.Done()
errs[i] = cl.CloseTarget(ctxt, t)
}(&wg, i, t)
}
wg.Wait()
for _, e := range errs {
if e != nil {
return e
}
}
// osx and linux applications do not automatically exit when all windows (ie, tabs)
// closed, so send SIGTERM.
//
// TODO: add other behavior here for more process options on shutdown?
if r.cmd != nil && r.cmd.Process != nil {
switch runtime.GOOS {
case "darwin", "linux":
return r.cmd.Process.Signal(syscall.SIGTERM)
}
}
return nil
}
// Wait waits for the previously started Chrome process to terminate, returning
// any encountered error.
func (r *Runner) Wait() error {
r.rw.RLock()
waiting := r.waiting
r.rw.RUnlock()
if waiting {
return ErrAlreadyWaiting
}
r.rw.Lock()
r.waiting = true
r.rw.Unlock()
defer func() {
r.rw.Lock()
r.waiting = false
r.rw.Unlock()
}()
return r.cmd.Wait()
}
// Port returns the port the process was launched with.
func (r *Runner) Port() int {
var port interface{}
var ok bool
port, ok = r.opts["remote-debugging-port"]
if !ok {
port, ok = r.opts["port"]
}
if !ok {
panic("expected either remote-debugging-port or port to be specified in command line options")
}
var p int
p, ok = port.(int)
if !ok {
panic("expected port to be type int")
}
return p
}
// Client returns a Chrome DevTools Protocol client for the running Chrome
// process.
func (r *Runner) Client(opts ...client.Option) *client.Client {
return client.New(append(opts,
client.URL(fmt.Sprintf("http://localhost:%d/json", r.Port())),
)...)
}
// Run starts a new Chrome process runner, using the provided context and
// command line options.
func Run(ctxt context.Context, opts ...CommandLineOption) (*Runner, error) {
var err error
// create
r, err := New(opts...)
if err != nil {
return nil, err
}
// start
if err = r.Start(ctxt); err != nil {
return nil, err
}
return r, nil
}
// CommandLineOption is a runner command line option.
//
// see: http://peter.sh/experiments/chromium-command-line-switches/
type CommandLineOption func(map[string]interface{}) error
// Flag is a generic command line option to pass a name=value flag to
// Chrome.
func Flag(name string, value interface{}) CommandLineOption {
return func(m map[string]interface{}) error {
m[name] = value
return nil
}
}
// Path sets the path to the Chrome executable and sets default run options for
// Chrome. This will also set the remote debugging port to 9222, and disable
// the first run / default browser check.
//
// Note: use ExecPath if you do not want to set other options.
func Path(path string) CommandLineOption {
return func(m map[string]interface{}) error {
m["exec-path"] = path
m["no-first-run"] = true
m["no-default-browser-check"] = true
m["remote-debugging-port"] = 9222
return nil
}
}
// ExecPath is a command line option to set the exec path.
func ExecPath(path string) CommandLineOption {
return Flag("exec-path", path)
}
// UserDataDir is the command line option to set the user data dir.
//
// Note: set this option to manually set the profile directory used by Chrome.
// When this is not set, then a default path will be created in the /tmp
// directory.
func UserDataDir(dir string) CommandLineOption {
return Flag("user-data-dir", dir)
}
// ProxyServer is the command line option to set the outbound proxy server.
func ProxyServer(proxy string) CommandLineOption {
return Flag("proxy-server", proxy)
}
// WindowSize is the command line option to set the initial window size.
func WindowSize(width, height int) CommandLineOption {
return Flag("window-size", fmt.Sprintf("%d,%d", width, height))
}
// UserAgent is the command line option to set the default User-Agent
// header.
func UserAgent(userAgent string) CommandLineOption {
return Flag("user-agent", userAgent)
}
// NoSandbox is the Chrome comamnd line option to disable the sandbox.
func NoSandbox(m map[string]interface{}) error {
return Flag("no-sandbox", true)(m)
}
// NoFirstRun is the Chrome comamnd line option to disable the first run
// dialog.
func NoFirstRun(m map[string]interface{}) error {
return Flag("no-first-run", true)(m)
}
// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the
// default browser check.
func NoDefaultBrowserCheck(m map[string]interface{}) error {
return Flag("no-default-browser-check", true)(m)
}
// RemoteDebuggingPort is the command line option to set the remote
// debugging port.
func RemoteDebuggingPort(port int) CommandLineOption {
return Flag("remote-debugging-port", port)
}
// Headless is the command line option to run in headless mode.
func Headless(m map[string]interface{}) error {
return Flag("headless", true)(m)
}
// DisableGPU is the command line option to disable the GPU process.
func DisableGPU(m map[string]interface{}) error {
return Flag("disable-gpu", true)(m)
}
// URL is the command line option to add a URL to open on process start.
//
// Note: this can be specified multiple times, and each URL will be opened in a
// new tab.
func URL(urlstr string) CommandLineOption {
return func(m map[string]interface{}) error {
var urls []string
if u, ok := m["url-opts"]; ok {
urls, ok = u.([]string)
if !ok {
return ErrInvalidURLOpts
}
}
m["url-opts"] = append(urls, urlstr)
return nil
}
}
// CmdOpt is a command line option to modify the underlying exec.Cmd
// prior to the call to exec.Cmd.Start in Run.
func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {
return func(m map[string]interface{}) error {
var opts []func(*exec.Cmd) error
if e, ok := m["cmd-opts"]; ok {
opts, ok = e.([]func(*exec.Cmd) error)
if !ok {
return ErrInvalidCmdOpts
}
}
m["cmd-opts"] = append(opts, o)
return nil
}
}
// ProcessOpt is a command line option to modify the child os.Process
// after the call to exec.Cmd.Start in Run.
func ProcessOpt(o func(*os.Process) error) CommandLineOption {
return func(m map[string]interface{}) error {
var opts []func(*os.Process) error
if e, ok := m["process-opts"]; ok {
opts, ok = e.([]func(*os.Process) error)
if !ok {
return ErrInvalidProcessOpts
}
}
m["process-opts"] = append(opts, o)
return nil
}
}
// LookChromeNames looks for the platform's DefaultChromeNames and any
// additional names using exec.LookPath, returning the first encountered
// location or the platform's DefaultChromePath if no names are found on the
// path.
func LookChromeNames(additional ...string) string {
for _, p := range append(additional, DefaultChromeNames...) {
path, err := exec.LookPath(p)
if err == nil {
return path
}
}
return DefaultChromePath
}

11
runner/runner_bsd.go
View File

@ -1,11 +0,0 @@
// +build darwin freebsd netbsd openbsd
package runner
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return nil
}

87
runner/runner_linux.go
View File

@ -1,87 +0,0 @@
// +build linux
package runner
import (
"os"
"os/exec"
"syscall"
"unsafe"
)
// ByteCount is a type byte count const.
type ByteCount uint64
// ByteCount values.
const (
Byte ByteCount = 1
Kilobyte ByteCount = 1024 * Byte
Megabyte ByteCount = 1024 * Kilobyte
Gigabyte ByteCount = 1024 * Megabyte
)
// prlimit invokes the system's prlimit call. Copied from Go source tree.
//
// Note: this needs either the CAP_SYS_RESOURCE capability, or the invoking
// process needs to have the same functional user and group as the pid being
// modified.
//
// see: man 2 prlimit
func prlimit(pid int, res int, newv, old *syscall.Rlimit) error {
_, _, err := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, uintptr(pid), uintptr(res), uintptr(unsafe.Pointer(newv)), uintptr(unsafe.Pointer(old)), 0, 0)
if err != 0 {
return err
}
return nil
}
// Rlimit is a Chrome command line option to set the soft rlimit value for res
// on a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child process
// has been started.
//
// see: man 2 prlimit
func Rlimit(res int, cur, max uint64) CommandLineOption {
return ProcessOpt(func(p *os.Process) error {
return prlimit(p.Pid, syscall.RLIMIT_AS, &syscall.Rlimit{
Cur: cur,
Max: max,
}, nil)
})
}
// LimitMemory is a Chrome command line option to set the soft memory limit for
// a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child
// process has been started.
func LimitMemory(mem ByteCount) CommandLineOption {
return Rlimit(syscall.RLIMIT_AS, uint64(mem), uint64(mem))
}
// LimitCoreDump is a Chrome command line option to set the soft core dump
// limit for a running Chrome process.
//
// Note: uses Linux prlimit system call, and is invoked after the child
// process has been started.
func LimitCoreDump(sz ByteCount) CommandLineOption {
return Rlimit(syscall.RLIMIT_CORE, uint64(sz), uint64(sz))
}
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return CmdOpt(func(c *exec.Cmd) error {
if c.SysProcAttr == nil {
c.SysProcAttr = new(syscall.SysProcAttr)
}
c.SysProcAttr.Pdeathsig = syscall.SIGKILL
return nil
})(m)
}

31
runner/runner_unix.go
View File

@ -1,31 +0,0 @@
// +build linux darwin freebsd netbsd openbsd
package runner
import (
"os/exec"
"syscall"
)
var (
// DefaultUserDataTmpDir is the default directory path for created user
// data directories.
defaultUserDataTmpDir = "/tmp"
)
// KillProcessGroup is a Chrome command line option that will instruct the
// invoked child Chrome process to terminate when the parent process (ie, the
// Go application) dies.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true and does nothing on Windows.
func KillProcessGroup(m map[string]interface{}) error {
return CmdOpt(func(c *exec.Cmd) error {
if c.SysProcAttr == nil {
c.SysProcAttr = new(syscall.SysProcAttr)
}
c.SysProcAttr.Setpgid = true
return nil
})(m)
}

26
runner/runner_windows.go
View File

@ -1,26 +0,0 @@
// +build windows
package runner
import "os"
var (
defaultUserDataTmpDir = os.Getenv("USERPROFILE") + `\AppData\Local`
)
// KillProcessGroup is a Chrome command line option that will instruct the
// invoked child Chrome process to terminate when the parent process (ie, the
// Go application) dies.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true and does nothing on Windows.
func KillProcessGroup(m map[string]interface{}) error {
return nil
}
// ForceKill is a Chrome command line option that forces Chrome to be killed
// when the parent is killed.
//
// Note: sets exec.Cmd.SysProcAttr.Setpgid = true (only for Linux)
func ForceKill(m map[string]interface{}) error {
return nil
}

3
sel.go
View File

@ -107,8 +107,7 @@ func (s *Selector) run(ctxt context.Context, h *TargetHandler) chan error {
return
}
err = s.after(ctxt, h, nodes...)
if err != nil {
if err := s.after(ctxt, h, nodes...); err != nil {
ch <- err
}
return

134
sel_test.go
View File

@ -9,132 +9,118 @@ import (
func TestWaitReady(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
var nodeIDs []cdp.NodeID
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, WaitReady("#input2", ByID))
if err != nil {
if err := Run(ctx, WaitReady("#input2", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
if err := Run(ctx, Value(nodeIDs, &value, ByNodeID)); err != nil {
t.Fatalf("got error: %v", err)
}
}
func TestWaitVisible(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
var nodeIDs []cdp.NodeID
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, WaitVisible("#input2", ByID))
if err != nil {
if err := Run(ctx, WaitVisible("#input2", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
if err := Run(ctx, Value(nodeIDs, &value, ByNodeID)); err != nil {
t.Fatalf("got error: %v", err)
}
}
func TestWaitNotVisible(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
var nodeIDs []cdp.NodeID
err := c.Run(defaultContext, NodeIDs("#input2", &nodeIDs, ByID))
err := Run(ctx, NodeIDs("#input2", &nodeIDs, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if len(nodeIDs) != 1 {
t.Errorf("expected to have exactly 1 node id: got %d", len(nodeIDs))
}
err = c.Run(defaultContext, Click("#button2", ByID))
if err != nil {
if err := Run(ctx, Click("#button2", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitNotVisible("#input2", ByID))
if err != nil {
if err := Run(ctx, WaitNotVisible("#input2", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value(nodeIDs, &value, ByNodeID))
if err != nil {
if err := Run(ctx, Value(nodeIDs, &value, ByNodeID)); err != nil {
t.Fatalf("got error: %v", err)
}
}
func TestWaitEnabled(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
var attr string
var ok bool
err := c.Run(defaultContext, AttributeValue("#select1", "disabled", &attr, &ok, ByID))
err := Run(ctx, AttributeValue("#select1", "disabled", &attr, &ok, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if !ok {
t.Fatal("expected element to be disabled")
}
err = c.Run(defaultContext, Click("#button3", ByID))
if err != nil {
if err := Run(ctx, Click("#button3", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, WaitEnabled("#select1", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, AttributeValue("#select1", "disabled", &attr, &ok, ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitEnabled("#select1", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, AttributeValue("#select1", "disabled", &attr, &ok, ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected element to be enabled")
}
err = c.Run(defaultContext, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"))
if err != nil {
if err := Run(ctx, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true")); err != nil {
t.Fatalf("got error: %v", err)
}
var value string
err = c.Run(defaultContext, Value("#select1", &value, ByID))
if err != nil {
if err := Run(ctx, Value("#select1", &value, ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if value != "foo" {
t.Fatalf("expected value to be foo, got: %s", value)
}
@ -143,43 +129,36 @@ func TestWaitEnabled(t *testing.T) {
func TestWaitSelected(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
err := c.Run(defaultContext, Click("#button3", ByID))
err := Run(ctx, Click("#button3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitEnabled("#select1", ByID))
if err != nil {
if err := Run(ctx, WaitEnabled("#select1", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
var attr string
ok := false
err = c.Run(defaultContext, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, &ok))
if err != nil {
if err := Run(ctx, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, &ok)); err != nil {
t.Fatalf("got error: %v", err)
}
if ok {
t.Fatal("expected element to be not selected")
}
err = c.Run(defaultContext, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true"))
if err != nil {
if err := Run(ctx, SetAttributeValue(`//*[@id="select1"]/option[1]`, "selected", "true")); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, WaitSelected(`//*[@id="select1"]/option[1]`)); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, nil)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitSelected(`//*[@id="select1"]/option[1]`))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, AttributeValue(`//*[@id="select1"]/option[1]`, "selected", &attr, nil))
if err != nil {
t.Fatalf("got error: %v", err)
}
if attr != "true" {
t.Fatal("expected element to be selected")
}
@ -188,33 +167,30 @@ func TestWaitSelected(t *testing.T) {
func TestWaitNotPresent(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
err := c.Run(defaultContext, WaitVisible("#input3", ByID))
err := Run(ctx, WaitVisible("#input3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, Click("#button4", ByID))
if err != nil {
if err := Run(ctx, Click("#button4", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
if err := Run(ctx, WaitNotPresent("#input3", ByID)); err != nil {
t.Fatalf("got error: %v", err)
}
err = c.Run(defaultContext, WaitNotPresent("#input3", ByID))
if err != nil {
t.Fatalf("got error: %v", err)
}
}
func TestAtLeast(t *testing.T) {
t.Parallel()
c := testAllocate(t, "js.html")
defer c.Release()
ctx, cancel := testAllocate(t, "js.html")
defer cancel()
var nodes []*cdp.Node
err := c.Run(defaultContext, Nodes("//input", &nodes, AtLeast(3)))
err := Run(ctx, Nodes("//input", &nodes, AtLeast(3)))
if err != nil {
t.Fatalf("got error: %v", err)
}

7
util.go
View File

@ -1,10 +1,17 @@
package chromedp
import (
"time"
"github.com/chromedp/cdproto"
"github.com/chromedp/cdproto/cdp"
)
const (
// DefaultCheckDuration is the default time to sleep between a check.
DefaultCheckDuration = 50 * time.Millisecond
)
// frameOp is a frame manipulation operation.
type frameOp func(*cdp.Frame)