8 Commits

Author SHA1 Message Date
Kenneth Shaw
34591780d9 Updating dependencies 2018-07-13 12:49:42 +07:00
Kenneth Shaw
53015e7d81 Changing Debugging => DevTools 2018-07-13 12:46:30 +07:00
Kenneth Shaw
74e379587b Minor code cleanup to client package before rewrite 2018-07-13 11:35:03 +07:00
Kenneth Shaw
4ae10864e4 Adding client API changes prior to package rewrite 2018-07-13 11:24:37 +07:00
Kenneth Shaw
d413f67302 Minor cleanup to client/gen.go 2018-07-13 11:00:35 +07:00
Kenneth Shaw
622c90c82c Cleaning up runner API prior to package rewrite 2018-07-13 10:57:20 +07:00
Kenneth Shaw
e051c4a982 Removing errors dependency in client and runner packages 2018-07-13 09:28:45 +07:00
Kenneth Shaw
0406fa8a8a Fixing issue with kb 2018-07-10 19:51:58 +07:00
16 changed files with 214 additions and 290 deletions

View File

@@ -1,102 +1,18 @@
# About chromedp [![Build Status][1]][2] [![Coverage Status][3]][4] # About chromedp [![Build Status][1]][2] [![Coverage Status][3]][4]
Package chromedp is a faster, simpler way to drive browsers in Go using the Package chromedp is a faster, simpler way to drive browsers supporting the
[Chrome Debugging Protocol][5] (for Chrome, Edge, Safari, etc) without external [Chrome DevTools Protocol][5] in Go using the without external dependencies
dependencies (ie, Selenium, PhantomJS, etc). (ie, Selenium, PhantomJS, etc).
**NOTE:** chromedp's API is currently unstable, and may change at a moments
notice. There are likely extremely bad bugs lurking in this code. **CAVEAT USER**.
## Installing ## Installing
Install in the usual way: Install in the usual Go way:
```sh ```sh
go get -u github.com/chromedp/chromedp go get -u github.com/chromedp/chromedp
``` ```
## Using ## Examples
Below is a simple Google search performed using chromedp (taken from
[examples/simple][6]):
This example shows logic for a simple search for a known website, clicking on
the right link, and then taking a screenshot of a specific element on the
loaded page and saving that to a local file on disk.
```go
// Command simple is a chromedp example demonstrating how to do a simple google
// search.
package main
import (
"context"
"fmt"
"io/ioutil"
"log"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp"
)
func main() {
var err error
// create context
ctxt, cancel := context.WithCancel(context.Background())
defer cancel()
// create chrome instance
c, err := chromedp.New(ctxt, chromedp.WithLog(log.Printf))
if err != nil {
log.Fatal(err)
}
// run task list
var site, res string
err = c.Run(ctxt, googleSearch("site:brank.as", "Home", &site, &res))
if err != nil {
log.Fatal(err)
}
// shutdown chrome
err = c.Shutdown(ctxt)
if err != nil {
log.Fatal(err)
}
// wait for chrome to finish
err = c.Wait()
if err != nil {
log.Fatal(err)
}
log.Printf("saved screenshot from search result listing `%s` (%s)", res, site)
}
func googleSearch(q, text string, site, res *string) chromedp.Tasks {
var buf []byte
sel := fmt.Sprintf(`//a[text()[contains(., '%s')]]`, text)
return chromedp.Tasks{
chromedp.Navigate(`https://www.google.com`),
chromedp.WaitVisible(`#hplogo`, chromedp.ByID),
chromedp.SendKeys(`#lst-ib`, q+"\n", chromedp.ByID),
chromedp.WaitVisible(`#res`, chromedp.ByID),
chromedp.Text(sel, res),
chromedp.Click(sel),
chromedp.WaitNotVisible(`.preloader-content`, chromedp.ByQuery),
chromedp.WaitVisible(`a[href*="twitter"]`, chromedp.ByQuery),
chromedp.Location(site),
chromedp.ScrollIntoView(`.banner-section.third-section`, chromedp.ByQuery),
chromedp.Sleep(2 * time.Second), // wait for animation to finish
chromedp.Screenshot(`.banner-section.third-section`, &buf, chromedp.ByQuery),
chromedp.ActionFunc(func(context.Context, cdp.Executor) error {
return ioutil.WriteFile("screenshot.png", buf, 0644)
}),
}
}
```
Please see the [examples][6] project for more examples. Please refer to the Please see the [examples][6] project for more examples. Please refer to the
[GoDoc API listing][7] for a summary of the API and Actions. [GoDoc API listing][7] for a summary of the API and Actions.
@@ -104,11 +20,11 @@ Please see the [examples][6] project for more examples. Please refer to the
## Resources ## Resources
* [chromedp: A New Way to Drive the Web][8] - GopherCon SG 2017 talk * [chromedp: A New Way to Drive the Web][8] - GopherCon SG 2017 talk
* [Chrome DevTools Protocol][5] - Chrome Debugging Protocol Domain documentation * [Chrome DevTools Protocol][5] - Chrome DevTools Protocol Domain documentation
* [chromedp examples][6] - various `chromedp` examples * [chromedp examples][6] - various `chromedp` examples
* [`github.com/chromedp/cdproto`][9] - GoDoc listing for the CDP domains used by `chromedp` * [`github.com/chromedp/cdproto`][9] - GoDoc listing for the CDP domains used by `chromedp`
* [`github.com/chromedp/cdproto-gen`][10] - tool used to generate `cdproto` * [`github.com/chromedp/cdproto-gen`][10] - tool used to generate `cdproto`
* [`github.com/chromedp/chromedp-proxy`][11] - a simple CDP proxy for logging/debugging CDP clients and browser instances * [`github.com/chromedp/chromedp-proxy`][11] - a simple CDP proxy for logging CDP clients and browsers
## TODO ## TODO

View File

@@ -1,9 +1,9 @@
// Package chromedp is a high level Chrome Debugging Protocol domain manager // Package chromedp is a high level Chrome DevTools Protocol client that
// that simplifies driving web browsers (Chrome, Safari, Edge, Android Web // simplifies driving browsers for scraping, unit testing, or profiling web
// Views, and others) for scraping, unit testing, or profiling web pages. // pages using the CDP.
// //
// chromedp requires no third-party dependencies (ie, Selenium), implementing // chromedp requires no third-party dependencies, implementing the async Chrome
// the async Chrome Debugging Protocol natively. // DevTools Protocol entirely in Go.
package chromedp package chromedp
import ( import (
@@ -35,8 +35,9 @@ const (
DefaultPoolEndPort = 10000 DefaultPoolEndPort = 10000
) )
// CDP contains information for managing a Chrome process runner, low level // CDP is the high-level Chrome DevTools Protocol browser manager, handling the
// JSON and websocket client, and associated network, page, and DOM handling. // browser process runner, WebSocket clients, associated targets, and network,
// page, and DOM events.
type CDP struct { type CDP struct {
// r is the chrome runner. // r is the chrome runner.
r *runner.Runner r *runner.Runner
@@ -90,7 +91,7 @@ func New(ctxt context.Context, opts ...Option) (*CDP, error) {
// watch handlers // watch handlers
if c.watch == nil { if c.watch == nil {
c.watch = c.r.WatchPageTargets(ctxt) c.watch = c.r.Client().WatchPageTargets(ctxt)
} }
go func() { go func() {
@@ -339,7 +340,7 @@ func (c *CDP) Run(ctxt context.Context, a Action) error {
return a.Do(ctxt, cur) return a.Do(ctxt, cur)
} }
// Option is a Chrome Debugging Protocol option. // Option is a Chrome DevTools Protocol option.
type Option func(*CDP) error type Option func(*CDP) error
// WithRunner is a CDP option to specify the underlying Chrome runner to // WithRunner is a CDP option to specify the underlying Chrome runner to

View File

@@ -4,7 +4,7 @@ import "fmt"
//go:generate easyjson -omit_empty -output_filename easyjson.go chrome.go //go:generate easyjson -omit_empty -output_filename easyjson.go chrome.go
// Chrome holds connection information for a Chrome, Edge, or Safari target. // Chrome holds connection information for a Chrome target.
// //
//easyjson:json //easyjson:json
type Chrome struct { type Chrome struct {
@@ -20,7 +20,7 @@ type Chrome struct {
// String satisfies the stringer interface. // String satisfies the stringer interface.
func (c Chrome) String() string { func (c Chrome) String() string {
return fmt.Sprintf("%s (`%s`)", c.ID, c.Title) return fmt.Sprintf("[%s]: %q", c.ID, c.Title)
} }
// GetID returns the target ID. // GetID returns the target ID.
@@ -33,6 +33,12 @@ func (c *Chrome) GetType() TargetType {
return c.Type return c.Type
} }
// GetDevtoolsURL returns the devtools frontend target URL, satisfying the
// domains.Target interface.
func (c *Chrome) GetDevtoolsURL() string {
return c.DevtoolsURL
}
// GetWebsocketURL provides the websocket URL for the target, satisfying the // GetWebsocketURL provides the websocket URL for the target, satisfying the
// domains.Target interface. // domains.Target interface.
func (c *Chrome) GetWebsocketURL() string { func (c *Chrome) GetWebsocketURL() string {

View File

@@ -1,5 +1,4 @@
// Package client provides the low level Chrome Debugging Protocol JSON types // Package client provides the low level Chrome DevTools Protocol client.
// and related funcs.
package client package client
//go:generate go run gen.go //go:generate go run gen.go
@@ -45,15 +44,16 @@ const (
ErrUnsupportedProtocolVersion Error = "unsupported protocol version" ErrUnsupportedProtocolVersion Error = "unsupported protocol version"
) )
// Target is the common interface for a Chrome Debugging Protocol target. // Target is the common interface for a Chrome DevTools Protocol target.
type Target interface { type Target interface {
String() string String() string
GetID() string GetID() string
GetType() TargetType GetType() TargetType
GetDevtoolsURL() string
GetWebsocketURL() string GetWebsocketURL() string
} }
// Client is a Chrome Debugging Protocol client. // Client is a Chrome DevTools Protocol client.
type Client struct { type Client struct {
url string url string
check time.Duration check time.Duration
@@ -63,7 +63,7 @@ type Client struct {
rw sync.RWMutex rw sync.RWMutex
} }
// New creates a new Chrome Debugging Protocol client. // New creates a new Chrome DevTools Protocol client.
func New(opts ...Option) *Client { func New(opts ...Option) *Client {
c := &Client{ c := &Client{
url: DefaultEndpoint, url: DefaultEndpoint,
@@ -120,8 +120,7 @@ func (c *Client) ListTargets(ctxt context.Context) ([]Target, error) {
var err error var err error
var l []json.RawMessage var l []json.RawMessage
err = c.doReq(ctxt, "list", &l) if err = c.doReq(ctxt, "list", &l); err != nil {
if err != nil {
return nil, err return nil, err
} }
@@ -200,8 +199,7 @@ func (c *Client) newTarget(ctxt context.Context, buf []byte) (Target, error) {
case "chrome", "chromium", "microsoft edge", "safari", "": case "chrome", "chromium", "microsoft edge", "safari", "":
x := new(Chrome) x := new(Chrome)
if buf != nil { if buf != nil {
err = easyjson.Unmarshal(buf, x) if err = easyjson.Unmarshal(buf, x); err != nil {
if err != nil {
return nil, err return nil, err
} }
} }
@@ -226,8 +224,7 @@ func (c *Client) NewPageTargetWithURL(ctxt context.Context, urlstr string) (Targ
u += "?" + urlstr u += "?" + urlstr
} }
err = c.doReq(ctxt, u, t) if err = c.doReq(ctxt, u, t); err != nil {
if err != nil {
return nil, err return nil, err
} }
@@ -251,23 +248,15 @@ func (c *Client) CloseTarget(ctxt context.Context, t Target) error {
// VersionInfo returns information about the remote debugging protocol. // VersionInfo returns information about the remote debugging protocol.
func (c *Client) VersionInfo(ctxt context.Context) (map[string]string, error) { func (c *Client) VersionInfo(ctxt context.Context) (map[string]string, error) {
var err error v := make(map[string]string)
if err := c.doReq(ctxt, "version", &v); err != nil {
v := map[string]string{}
err = c.doReq(ctxt, "version", &v)
if err != nil {
return nil, err return nil, err
} }
return v, nil return v, nil
} }
// WatchPageTargets watches for new page targets. // WatchPageTargets watches for new page targets.
func (c *Client) WatchPageTargets(ctxt context.Context) <-chan Target { func (c *Client) WatchPageTargets(ctxt context.Context) <-chan Target {
if ctxt == nil {
ctxt = context.Background()
}
ch := make(chan Target) ch := make(chan Target)
go func() { go func() {
defer close(ch) defer close(ch)
@@ -311,10 +300,11 @@ func (c *Client) WatchPageTargets(ctxt context.Context) <-chan Target {
return ch return ch
} }
// Option is a Chrome Debugging Protocol client option. // Option is a Chrome DevTools Protocol client option.
type Option func(*Client) type Option func(*Client)
// URL is a client option to specify the remote Chrome instance to connect to. // URL is a client option to specify the remote Chrome DevTools Protocol
// instance to connect to.
func URL(urlstr string) Option { func URL(urlstr string) Option {
return func(c *Client) { return func(c *Client) {
// since chrome 66+, dev tools requires the host name to be either an // since chrome 66+, dev tools requires the host name to be either an

View File

@@ -26,17 +26,26 @@ const (
var ( var (
flagOut = flag.String("out", "targettype.go", "out file") flagOut = flag.String("out", "targettype.go", "out file")
typeAsStringRE = regexp.MustCompile(`type_as_string\s+==\s+"([^"]+)"`)
) )
func main() { func main() {
flag.Parse() flag.Parse()
if err := run(); err != nil {
log.Fatal(err)
}
}
var typeAsStringRE = regexp.MustCompile(`type_as_string\s+==\s+"([^"]+)"`)
// run executes the generator.
func run() error {
var err error
// grab source // grab source
buf, err := grab(devtoolsHTTPClientCc) buf, err := grab(devtoolsHTTPClientCc)
if err != nil { if err != nil {
log.Fatal(err) return err
} }
// find names // find names
@@ -55,15 +64,11 @@ func main() {
decodeVals += fmt.Sprintf("case %s:\n*tt=%s\n", name, name) decodeVals += fmt.Sprintf("case %s:\n*tt=%s\n", name, name)
} }
err = ioutil.WriteFile(*flagOut, []byte(fmt.Sprintf(targetTypeSrc, constVals, decodeVals)), 0644) if err = ioutil.WriteFile(*flagOut, []byte(fmt.Sprintf(targetTypeSrc, constVals, decodeVals)), 0644); err != nil {
if err != nil { return err
log.Fatal(err)
} }
err = exec.Command("gofmt", "-w", "-s", *flagOut).Run() return exec.Command("gofmt", "-w", "-s", *flagOut).Run()
if err != nil {
log.Fatal(err)
}
} }
// grab retrieves a file from the chromium source code. // grab retrieves a file from the chromium source code.
@@ -93,7 +98,6 @@ const (
// Code generated by gen.go. DO NOT EDIT. // Code generated by gen.go. DO NOT EDIT.
import ( import (
// "errors"
easyjson "github.com/mailru/easyjson" easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer" jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter" jwriter "github.com/mailru/easyjson/jwriter"
@@ -129,7 +133,6 @@ func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
%s %s
default: default:
// in.AddError(errors.New("unknown TargetType"))
*tt = z *tt = z
} }
} }

View File

@@ -3,7 +3,6 @@ package client
// Code generated by gen.go. DO NOT EDIT. // Code generated by gen.go. DO NOT EDIT.
import ( import (
// "errors"
easyjson "github.com/mailru/easyjson" easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer" jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter" jwriter "github.com/mailru/easyjson/jwriter"
@@ -70,7 +69,6 @@ func (tt *TargetType) UnmarshalEasyJSON(in *jlexer.Lexer) {
*tt = Worker *tt = Worker
default: default:
// in.AddError(errors.New("unknown TargetType"))
*tt = z *tt = z
} }
} }

View File

@@ -6,7 +6,7 @@ import (
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
) )
const ( var (
// DefaultReadBufferSize is the default maximum read buffer size. // DefaultReadBufferSize is the default maximum read buffer size.
DefaultReadBufferSize = 25 * 1024 * 1024 DefaultReadBufferSize = 25 * 1024 * 1024
@@ -14,7 +14,7 @@ const (
DefaultWriteBufferSize = 10 * 1024 * 1024 DefaultWriteBufferSize = 10 * 1024 * 1024
) )
// Transport is the common interface to send/receive messages. // Transport is the common interface to send/receive messages to a target.
type Transport interface { type Transport interface {
Read() ([]byte, error) Read() ([]byte, error)
Write([]byte) error Write([]byte) error
@@ -43,7 +43,7 @@ func (c *Conn) Write(buf []byte) error {
// Dial dials the specified target's websocket URL. // Dial dials the specified target's websocket URL.
// //
// Note: uses gorilla/websocket. // Note: uses gorilla/websocket.
func Dial(t Target, opts ...DialOption) (Transport, error) { func Dial(urlstr string, opts ...DialOption) (Transport, error) {
d := &websocket.Dialer{ d := &websocket.Dialer{
ReadBufferSize: DefaultReadBufferSize, ReadBufferSize: DefaultReadBufferSize,
WriteBufferSize: DefaultWriteBufferSize, WriteBufferSize: DefaultWriteBufferSize,
@@ -55,7 +55,7 @@ func Dial(t Target, opts ...DialOption) (Transport, error) {
} }
// connect // connect
conn, _, err := d.Dial(t.GetWebsocketURL(), nil) conn, _, err := d.Dial(urlstr, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -65,5 +65,3 @@ func Dial(t Target, opts ...DialOption) (Transport, error) {
// DialOption is a dial option. // DialOption is a dial option.
type DialOption func(*websocket.Dialer) type DialOption func(*websocket.Dialer)
// TODO: add dial options ...

4
go.mod
View File

@@ -1,10 +1,10 @@
module github.com/chromedp/chromedp module github.com/chromedp/chromedp
require ( require (
github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3 github.com/chromedp/cdproto v0.0.0-20180713053126-e314dc107013
github.com/disintegration/imaging v1.4.2 github.com/disintegration/imaging v1.4.2
github.com/gorilla/websocket v1.2.0 github.com/gorilla/websocket v1.2.0
github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794 github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794
github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856 github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856
golang.org/x/image v0.0.0-20180628062038-cc896f830ced golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81
) )

10
go.sum
View File

@@ -1,9 +1,19 @@
github.com/chromedp/cdproto v0.0.0-20180522032958-55db67b53f25/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs= github.com/chromedp/cdproto v0.0.0-20180522032958-55db67b53f25/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs=
github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3 h1:4b4LwyHW4sf6zXZqWsKMYoICFLWGhaqwpEHlnQMN5SE=
github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs= github.com/chromedp/cdproto v0.0.0-20180703215205-c125a34ea3b3/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs=
github.com/chromedp/cdproto v0.0.0-20180713053126-e314dc107013 h1:8nmuTwCseJcww39MvVHI59223+PxSzn6g3cl8ChF0/4=
github.com/chromedp/cdproto v0.0.0-20180713053126-e314dc107013/go.mod h1:C2GPAraqdt1KfZU7aSmx1XUgarNq/3JmxevQkmCjOVs=
github.com/disintegration/imaging v1.4.2 h1:BSVxoYQ2NfLdvIGCDD8GHgBV5K0FCEsc0d/6FxQII3I=
github.com/disintegration/imaging v1.4.2/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU= github.com/disintegration/imaging v1.4.2/go.mod h1:9B/deIUIrliYkyMTuXJd6OUFLcrZ2tf+3Qlwnaf/CjU=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794 h1:hgWKTlyruPI7k8W+0FmTMLf+8d2KPxyzTxsfDDQhNp8=
github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ= github.com/knq/sysutil v0.0.0-20180306023629-0218e141a794/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856 h1:hOnidOuIWNsFRPcxxStGeN3NNm4n4+w6KJ9cVJIh70o=
github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180606163543-3fdea8d05856/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
golang.org/x/image v0.0.0-20180403161127-f315e4403028/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20180403161127-f315e4403028/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20180628062038-cc896f830ced h1:2QsAEqOy4Mp+V4HL2Wr1iBNpZWaL72EvTO4oj5bmr5w=
golang.org/x/image v0.0.0-20180628062038-cc896f830ced/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20180628062038-cc896f830ced/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=

View File

@@ -24,7 +24,7 @@ import (
"github.com/chromedp/chromedp/client" "github.com/chromedp/chromedp/client"
) )
// TargetHandler manages a Chrome Debugging Protocol target. // TargetHandler manages a Chrome DevTools Protocol target.
type TargetHandler struct { type TargetHandler struct {
conn client.Transport conn client.Transport
@@ -64,7 +64,7 @@ type TargetHandler struct {
// NewTargetHandler creates a new handler for the specified client target. // NewTargetHandler creates a new handler for the specified client target.
func NewTargetHandler(t client.Target, logf, debugf, errf func(string, ...interface{})) (*TargetHandler, error) { func NewTargetHandler(t client.Target, logf, debugf, errf func(string, ...interface{})) (*TargetHandler, error) {
conn, err := client.Dial(t) conn, err := client.Dial(t.GetWebsocketURL())
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@@ -5,6 +5,7 @@ package kb
//go:generate go run gen.go -out keys.go -pkg kb //go:generate go run gen.go -out keys.go -pkg kb
import ( import (
"runtime"
"unicode" "unicode"
"github.com/chromedp/cdproto/input" "github.com/chromedp/cdproto/input"
@@ -97,6 +98,9 @@ func Encode(r rune) []*input.DispatchKeyEventParams {
NativeVirtualKeyCode: v.Native, NativeVirtualKeyCode: v.Native,
WindowsVirtualKeyCode: v.Windows, WindowsVirtualKeyCode: v.Windows,
} }
if runtime.GOOS == "darwin" {
keyDown.NativeVirtualKeyCode = 0
}
if v.Shift { if v.Shift {
keyDown.Modifiers |= input.ModifierShift keyDown.Modifiers |= input.ModifierShift
} }

View File

@@ -74,7 +74,11 @@ func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption)
// create runner // create runner
r.r, err = runner.New(append([]runner.CommandLineOption{ r.r, err = runner.New(append([]runner.CommandLineOption{
runner.HeadlessPathPort("", r.port), runner.ExecPath(runner.LookChromeNames("headless_shell")),
runner.RemoteDebuggingPort(r.port),
runner.NoDefaultBrowserCheck,
runner.NoFirstRun,
runner.Headless,
}, opts...)...) }, opts...)...)
if err != nil { if err != nil {
defer r.Release() defer r.Release()

View File

@@ -3,10 +3,11 @@
package runner package runner
const ( const (
// DefaultChromePath is the default path to the Chrome application. // DefaultChromePath is the default path to use for Chrome if the
// executable is not in $PATH.
DefaultChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` DefaultChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`
) )
func findChromePath() string { // DefaultChromeNames are the default Chrome executable names to look for in
return DefaultChromePath // $PATH.
} var DefaultChromeNames []string

View File

@@ -2,30 +2,18 @@
package runner package runner
import "os/exec"
const ( const (
// DefaultChromePath is the default path to the google-chrome executable if // DefaultChromePath is the default path to use for Chrome if the
// a variant cannot be found on $PATH. // executable is not in $PATH.
DefaultChromePath = "/usr/bin/google-chrome" DefaultChromePath = "/usr/bin/google-chrome"
) )
// chromeNames are the Chrome executable names to search for in the path. // DefaultChromeNames are the default Chrome executable names to look for in
var chromeNames = []string{ // $PATH.
var DefaultChromeNames = []string{
"google-chrome", "google-chrome",
"chromium-browser", "chromium-browser",
"chromium", "chromium",
"google-chrome-beta", "google-chrome-beta",
"google-chrome-unstable", "google-chrome-unstable",
} }
func findChromePath() string {
for _, p := range chromeNames {
path, err := exec.LookPath(p)
if err == nil {
return path
}
}
return DefaultChromePath
}

View File

@@ -2,32 +2,12 @@
package runner package runner
import "os/exec"
const ( const (
// DefaultChromePath is the default path to use for Google Chrome if the // DefaultChromePath is the default path to use for Chrome if the
// executable is not in %PATH%. // executable is not in %PATH%.
DefaultChromePath = `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe` DefaultChromePath = `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`
// DefaultEdgeDiagnosticsAdapterPath is the default path to use for the
// Microsoft Edge Diagnostics Adapter if the executable is not in %PATH%.
DefaultEdgeDiagnosticsAdapterPath = `c:\Edge\EdgeDiagnosticsAdapter\x64\EdgeDiagnosticsAdapter.exe`
) )
func findChromePath() string { // DefaultChromeNames are the default Chrome executable names to look for in
path, err := exec.LookPath(`chrome.exe`) // %PATH%.
if err == nil { var DefaultChromeNames = []string{`chrome.exe`}
return path
}
return DefaultChromePath
}
func findEdgePath() string {
path, err := exec.LookPath(`EdgeDiagnosticsAdapter.exe`)
if err == nil {
return path
}
return DefaultEdgeDiagnosticsAdapterPath
}

View File

@@ -3,10 +3,8 @@ package runner
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/url"
"os" "os"
"os/exec" "os/exec"
"regexp" "regexp"
@@ -22,6 +20,35 @@ const (
DefaultUserDataDirPrefix = "chromedp-runner.%d." DefaultUserDataDirPrefix = "chromedp-runner.%d."
) )
// Error is a runner error.
type Error string
// Error satisfies the error interface.
func (err Error) Error() string {
return string(err)
}
// Error values.
const (
// ErrAlreadyStarted is the already started error.
ErrAlreadyStarted Error = "already started"
// ErrAlreadyWaiting is the already waiting error.
ErrAlreadyWaiting Error = "already waiting"
// ErrInvalidURLs is the invalid url-opts error.
ErrInvalidURLOpts Error = "invalid url-opts"
// ErrInvalidCmdOpts is the invalid cmd-opts error.
ErrInvalidCmdOpts Error = "invalid cmd-opts"
// ErrInvalidProcessOpts is the invalid process-opts error.
ErrInvalidProcessOpts Error = "invalid process-opts"
// ErrInvalidExecPath is the invalid exec-path error.
ErrInvalidExecPath Error = "invalid exec-path"
)
// Runner holds information about a running Chrome process. // Runner holds information about a running Chrome process.
type Runner struct { type Runner struct {
opts map[string]interface{} opts map[string]interface{}
@@ -34,19 +61,18 @@ type Runner struct {
func New(opts ...CommandLineOption) (*Runner, error) { func New(opts ...CommandLineOption) (*Runner, error) {
var err error var err error
cliOpts := map[string]interface{}{} cliOpts := make(map[string]interface{})
// apply opts // apply opts
for _, o := range opts { for _, o := range opts {
err = o(cliOpts) if err = o(cliOpts); err != nil {
if err != nil {
return nil, err return nil, err
} }
} }
// set default Chrome options if exec-path not provided // set default Chrome options if exec-path not provided
if _, ok := cliOpts["exec-path"]; !ok { if _, ok := cliOpts["exec-path"]; !ok {
cliOpts["exec-path"] = findChromePath() cliOpts["exec-path"] = LookChromeNames()
for k, v := range map[string]interface{}{ for k, v := range map[string]interface{}{
"no-first-run": true, "no-first-run": true,
"no-default-browser-check": true, "no-default-browser-check": true,
@@ -61,8 +87,7 @@ func New(opts ...CommandLineOption) (*Runner, error) {
// add KillProcessGroup and ForceKill if no other cmd opts provided // add KillProcessGroup and ForceKill if no other cmd opts provided
if _, ok := cliOpts["cmd-opts"]; !ok { if _, ok := cliOpts["cmd-opts"]; !ok {
for _, o := range []CommandLineOption{KillProcessGroup, ForceKill} { for _, o := range []CommandLineOption{KillProcessGroup, ForceKill} {
err = o(cliOpts) if err = o(cliOpts); err != nil {
if err != nil {
return nil, err return nil, err
} }
} }
@@ -79,9 +104,9 @@ var cliOptRE = regexp.MustCompile(`^[a-z0-9\-]+$`)
// buildOpts generates the command line options for Chrome. // buildOpts generates the command line options for Chrome.
func (r *Runner) buildOpts() []string { func (r *Runner) buildOpts() []string {
var opts []string var opts []string
var urls []string
// process options // process opts
var urlstr string
for k, v := range r.opts { for k, v := range r.opts {
if !cliOptRE.MatchString(k) || v == nil { if !cliOptRE.MatchString(k) || v == nil {
continue continue
@@ -91,8 +116,8 @@ func (r *Runner) buildOpts() []string {
case "exec-path", "cmd-opts", "process-opts": case "exec-path", "cmd-opts", "process-opts":
continue continue
case "start-url": case "url-opts":
urlstr = v.(string) urls = v.([]string)
default: default:
switch z := v.(type) { switch z := v.(type) {
@@ -110,16 +135,16 @@ func (r *Runner) buildOpts() []string {
} }
} }
if urlstr == "" { if urls == nil {
urlstr = "about:blank" urls = append(urls, "about:blank")
} }
return append(opts, urlstr) return append(opts, urls...)
} }
// Start starts a Chrome process using the specified context. The Chrome // Start starts a Chrome process using the specified context. The Chrome
// process can be terminated by closing the passed context. // process can be terminated by closing the passed context.
func (r *Runner) Start(ctxt context.Context) error { func (r *Runner) Start(ctxt context.Context, opts ...string) error {
var err error var err error
var ok bool var ok bool
@@ -128,12 +153,7 @@ func (r *Runner) Start(ctxt context.Context) error {
r.rw.RUnlock() r.rw.RUnlock()
if cmd != nil { if cmd != nil {
return errors.New("already started") return ErrAlreadyStarted
}
// setup context
if ctxt == nil {
ctxt = context.Background()
} }
// set user data dir, if not provided // set user data dir, if not provided
@@ -147,36 +167,41 @@ func (r *Runner) Start(ctxt context.Context) error {
} }
} }
// ensure exec-path set // get exec path
execPath, ok := r.opts["exec-path"] var execPath string
if p, ok := r.opts["exec-path"]; ok {
execPath, ok = p.(string)
if !ok { if !ok {
return errors.New("exec-path command line option not set, or chrome executable not found in $PATH") return ErrInvalidExecPath
}
}
// ensure execPath is valid
if execPath == "" {
return ErrInvalidExecPath
} }
// create cmd // create cmd
r.cmd = exec.CommandContext(ctxt, execPath.(string), r.buildOpts()...) r.cmd = exec.CommandContext(ctxt, execPath, append(r.buildOpts(), opts...)...)
// apply cmd opts // apply cmd opts
if cmdOpts, ok := r.opts["cmd-opts"]; ok { if cmdOpts, ok := r.opts["cmd-opts"]; ok {
for _, co := range cmdOpts.([]func(*exec.Cmd) error) { for _, co := range cmdOpts.([]func(*exec.Cmd) error) {
err = co(r.cmd) if err = co(r.cmd); err != nil {
if err != nil {
return err return err
} }
} }
} }
// start process // start process
err = r.cmd.Start() if err = r.cmd.Start(); err != nil {
if err != nil {
return err return err
} }
// apply process opts // apply process opts
if processOpts, ok := r.opts["process-opts"]; ok { if processOpts, ok := r.opts["process-opts"]; ok {
for _, po := range processOpts.([]func(*os.Process) error) { for _, po := range processOpts.([]func(*os.Process) error) {
err = po(r.cmd.Process) if err = po(r.cmd.Process); err != nil {
if err != nil {
// TODO: do something better here, as we want to kill // TODO: do something better here, as we want to kill
// the child process, do cleanup, etc. // the child process, do cleanup, etc.
panic(err) panic(err)
@@ -235,7 +260,7 @@ func (r *Runner) Wait() error {
r.rw.RUnlock() r.rw.RUnlock()
if waiting { if waiting {
return errors.New("already waiting") return ErrAlreadyWaiting
} }
r.rw.Lock() r.rw.Lock()
@@ -272,7 +297,7 @@ func (r *Runner) Port() int {
return p return p
} }
// Client returns a Chrome Debugging Protocol client for the running Chrome // Client returns a Chrome DevTools Protocol client for the running Chrome
// process. // process.
func (r *Runner) Client(opts ...client.Option) *client.Client { func (r *Runner) Client(opts ...client.Option) *client.Client {
return client.New(append(opts, return client.New(append(opts,
@@ -280,12 +305,6 @@ func (r *Runner) Client(opts ...client.Option) *client.Client {
)...) )...)
} }
// WatchPageTargets returns a channel that will receive new page targets as
// they are created.
func (r *Runner) WatchPageTargets(ctxt context.Context, opts ...client.Option) <-chan client.Target {
return r.Client(opts...).WatchPageTargets(ctxt)
}
// Run starts a new Chrome process runner, using the provided context and // Run starts a new Chrome process runner, using the provided context and
// command line options. // command line options.
func Run(ctxt context.Context, opts ...CommandLineOption) (*Runner, error) { func Run(ctxt context.Context, opts ...CommandLineOption) (*Runner, error) {
@@ -298,20 +317,19 @@ func Run(ctxt context.Context, opts ...CommandLineOption) (*Runner, error) {
} }
// start // start
err = r.Start(ctxt) if err = r.Start(ctxt); err != nil {
if err != nil {
return nil, err return nil, err
} }
return r, nil return r, nil
} }
// CommandLineOption is a Chrome command line option. // CommandLineOption is a runner command line option.
// //
// see: http://peter.sh/experiments/chromium-command-line-switches/ // see: http://peter.sh/experiments/chromium-command-line-switches/
type CommandLineOption func(map[string]interface{}) error type CommandLineOption func(map[string]interface{}) error
// Flag is a generic Chrome command line option to pass a name=value flag to // Flag is a generic command line option to pass a name=value flag to
// Chrome. // Chrome.
func Flag(name string, value interface{}) CommandLineOption { func Flag(name string, value interface{}) CommandLineOption {
return func(m map[string]interface{}) error { return func(m map[string]interface{}) error {
@@ -335,33 +353,12 @@ func Path(path string) CommandLineOption {
} }
} }
// HeadlessPathPort is the Chrome command line option to set the default // ExecPath is a command line option to set the exec path.
// settings for running the headless_shell executable. If path is empty, then
// an attempt will be made to find headless_shell on the path.
func HeadlessPathPort(path string, port int) CommandLineOption {
if path == "" {
path, _ = exec.LookPath("headless_shell")
}
return func(m map[string]interface{}) error {
m["exec-path"] = path
m["remote-debugging-port"] = port
m["headless"] = true
return nil
}
}
// ExecPath is a Chrome command line option to set the exec path.
func ExecPath(path string) CommandLineOption { func ExecPath(path string) CommandLineOption {
return Flag("exec-path", path) return Flag("exec-path", path)
} }
// Port is the Chrome command line option to set the remote debugging port. // UserDataDir is the command line option to set the user data dir.
func Port(port int) CommandLineOption {
return Flag("remote-debugging-port", port)
}
// UserDataDir is the Chrome command line option to set the user data dir.
// //
// Note: set this option to manually set the profile directory used by Chrome. // Note: set this option to manually set the profile directory used by Chrome.
// When this is not set, then a default path will be created in the /tmp // When this is not set, then a default path will be created in the /tmp
@@ -370,27 +367,17 @@ func UserDataDir(dir string) CommandLineOption {
return Flag("user-data-dir", dir) return Flag("user-data-dir", dir)
} }
// StartURL is the Chrome command line option to set the initial URL. // ProxyServer is the command line option to set the outbound proxy server.
func StartURL(urlstr string) CommandLineOption { func ProxyServer(proxy string) CommandLineOption {
return Flag("start-url", urlstr)
}
// Proxy is the Chrome command line option to set the outbound proxy.
func Proxy(proxy string) CommandLineOption {
return Flag("proxy-server", proxy) return Flag("proxy-server", proxy)
} }
// ProxyPacURL is the Chrome command line option to set the URL of a proxy PAC file. // WindowSize is the command line option to set the initial window size.
func ProxyPacURL(pacURL url.URL) CommandLineOption {
return Flag("proxy-pac-url", pacURL.String())
}
// WindowSize is the Chrome command line option to set the initial window size.
func WindowSize(width, height int) CommandLineOption { func WindowSize(width, height int) CommandLineOption {
return Flag("window-size", fmt.Sprintf("%d,%d", width, height)) return Flag("window-size", fmt.Sprintf("%d,%d", width, height))
} }
// UserAgent is the Chrome command line option to set the default User-Agent // UserAgent is the command line option to set the default User-Agent
// header. // header.
func UserAgent(userAgent string) CommandLineOption { func UserAgent(userAgent string) CommandLineOption {
return Flag("user-agent", userAgent) return Flag("user-agent", userAgent)
@@ -413,45 +400,83 @@ func NoDefaultBrowserCheck(m map[string]interface{}) error {
return Flag("no-default-browser-check", true)(m) return Flag("no-default-browser-check", true)(m)
} }
// DisableGPU is the Chrome command line option to disable the GPU process. // RemoteDebuggingPort is the command line option to set the remote
// debugging port.
func RemoteDebuggingPort(port int) CommandLineOption {
return Flag("remote-debugging-port", port)
}
// Headless is the command line option to run in headless mode.
func Headless(m map[string]interface{}) error {
return Flag("headless", true)(m)
}
// DisableGPU is the command line option to disable the GPU process.
func DisableGPU(m map[string]interface{}) error { func DisableGPU(m map[string]interface{}) error {
return Flag("disable-gpu", true)(m) return Flag("disable-gpu", true)(m)
} }
// CmdOpt is a Chrome command line option to modify the underlying exec.Cmd // URL is the command line option to add a URL to open on process start.
// prior to invocation. //
// Note: this can be specified multiple times, and each URL will be opened in a
// new tab.
func URL(urlstr string) CommandLineOption {
return func(m map[string]interface{}) error {
var urls []string
if u, ok := m["url-opts"]; ok {
urls, ok = u.([]string)
if !ok {
return ErrInvalidURLOpts
}
}
m["url-opts"] = append(urls, urlstr)
return nil
}
}
// CmdOpt is a command line option to modify the underlying exec.Cmd
// prior to the call to exec.Cmd.Start in Run.
func CmdOpt(o func(*exec.Cmd) error) CommandLineOption { func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {
return func(m map[string]interface{}) error { return func(m map[string]interface{}) error {
var opts []func(*exec.Cmd) error var opts []func(*exec.Cmd) error
if e, ok := m["cmd-opts"]; ok { if e, ok := m["cmd-opts"]; ok {
opts, ok = e.([]func(*exec.Cmd) error) opts, ok = e.([]func(*exec.Cmd) error)
if !ok { if !ok {
return errors.New("cmd-opts is in invalid state") return ErrInvalidCmdOpts
} }
} }
m["cmd-opts"] = append(opts, o) m["cmd-opts"] = append(opts, o)
return nil return nil
} }
} }
// ProcessOpt is a Chrome command line option to modify the child os.Process // ProcessOpt is a command line option to modify the child os.Process
// after started exec.Cmd.Start. // after the call to exec.Cmd.Start in Run.
func ProcessOpt(o func(*os.Process) error) CommandLineOption { func ProcessOpt(o func(*os.Process) error) CommandLineOption {
return func(m map[string]interface{}) error { return func(m map[string]interface{}) error {
var opts []func(*os.Process) error var opts []func(*os.Process) error
if e, ok := m["process-opts"]; ok { if e, ok := m["process-opts"]; ok {
opts, ok = e.([]func(*os.Process) error) opts, ok = e.([]func(*os.Process) error)
if !ok { if !ok {
return errors.New("process-opts is in invalid state") return ErrInvalidProcessOpts
} }
} }
m["process-opts"] = append(opts, o) m["process-opts"] = append(opts, o)
return nil return nil
} }
} }
// LookChromeNames looks for the platform's DefaultChromeNames and any
// additional names using exec.LookPath, returning the first encountered
// location or the platform's DefaultChromePath if no names are found on the
// path.
func LookChromeNames(additional ...string) string {
for _, p := range append(additional, DefaultChromeNames...) {
path, err := exec.LookPath(p)
if err == nil {
return path
}
}
return DefaultChromePath
}