diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..dedc9dc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: go +go: + - 1.7 + - tip +before_install: + - mkdir -p /headless_shell + - wget -O /headless_shell/headless_shell.tar.bz2 https://storage.googleapis.com/docker-chrome-headless/headless_shell.tar.bz2 + - tar -jxf /headless_shell/headless_shell.tar.bz2 -C /headless_shell + - go get github.com/mattn/goveralls +script: + - CHROMEDP_NO_SANDBOX=true go test -v -coverprofile=coverage.out + - goveralls -service=travis-ci -coverprofile=coverage.out diff --git a/chromedp.go b/chromedp.go index f9c88ec..5cd426b 100644 --- a/chromedp.go +++ b/chromedp.go @@ -68,13 +68,6 @@ func New(ctxt context.Context, opts ...Option) (*CDP, error) { } } - // setup context - if ctxt == nil { - var cancel func() - ctxt, cancel = context.WithCancel(context.Background()) - defer cancel() - } - // check for supplied runner, if none then create one if c.r == nil && c.watch == nil { c.r, err = runner.Run(ctxt, c.opts...) @@ -312,30 +305,30 @@ func (c *CDP) NewTarget(id *string, opts ...client.Option) Action { // NewTargetWithURL creates a new Chrome target, sets it as the active target, // and then navigates to the specified url. -func (c *CDP) NewTargetWithURL(urlstr string, id *string, opts ...client.Option) Action { - return ActionFunc(func(ctxt context.Context, h cdp.FrameHandler) error { - n, err := c.newTarget(ctxt, opts...) - if err != nil { - return err - } - - l := c.GetHandlerByID(n) - if l == nil { - return errors.New("could not retrieve newly created target") - } - - /*err = Navigate(l, urlstr).Do(ctxt) - if err != nil { - return err - } - - if id != nil { - *id = n - }*/ - - return nil - }) -} +//func (c *CDP) NewTargetWithURL(urlstr string, id *string, opts ...client.Option) Action { +// return ActionFunc(func(ctxt context.Context, h cdp.FrameHandler) error { +// n, err := c.newTarget(ctxt, opts...) +// if err != nil { +// return err +// } +// +// l := c.GetHandlerByID(n) +// if l == nil { +// return errors.New("could not retrieve newly created target") +// } +// +// /*err = Navigate(l, urlstr).Do(ctxt) +// if err != nil { +// return err +// } +// +// if id != nil { +// *id = n +// }*/ +// +// return nil +// }) +//} // CloseByIndex closes the Chrome target with specified index i. func (c *CDP) CloseByIndex(i int) Action { diff --git a/chromedp_test.go b/chromedp_test.go new file mode 100644 index 0000000..ca65103 --- /dev/null +++ b/chromedp_test.go @@ -0,0 +1,61 @@ +package chromedp + +import ( + "context" + "log" + "os" + "strings" + "testing" +) + +var pool *Pool + +var defaultContext = context.Background() + +func TestMain(m *testing.M) { + var err error + + pool, err = NewPool() + if err != nil { + log.Fatal(err) + } + + code := m.Run() + + err = pool.Shutdown() + if err != nil { + log.Fatal(err) + } + + os.Exit(code) +} + +func TestNavigate(t *testing.T) { + var err error + + c, err := pool.Allocate(defaultContext) + if err != nil { + t.Fatal(err) + } + defer c.Release() + + err = c.Run(defaultContext, Navigate("https://www.google.com/")) + if err != nil { + t.Fatal(err) + } + + err = c.Run(defaultContext, WaitVisible(`#hplogo`, ByID)) + if err != nil { + t.Fatal(err) + } + + var urlstr string + err = c.Run(defaultContext, Location(&urlstr)) + if err != nil { + t.Fatal(err) + } + + if !strings.HasPrefix(urlstr, "https://www.google.") { + t.Errorf("expected to be on google, got: %v", urlstr) + } +} diff --git a/pool.go b/pool.go new file mode 100644 index 0000000..26e9c33 --- /dev/null +++ b/pool.go @@ -0,0 +1,183 @@ +package chromedp + +import ( + "context" + "fmt" + "sync" + + "github.com/knq/chromedp/runner" +) + +const ( + // DefaultStartPort is the default start port number. + DefaultStartPort = 9000 + + // DefaultEndPort is the default end port number. + DefaultEndPort = 10000 +) + +// Pool provides a pool of running Chrome processes. +type Pool struct { + // start is the start port. + start int + + // end is the end port. + end int + + // res are the running chrome resources. + res map[int]*Res + + rw sync.RWMutex +} + +// NewPool creates a new Chrome runner pool. +func NewPool(opts ...PoolOption) (*Pool, error) { + var err error + + p := &Pool{ + start: DefaultStartPort, + end: DefaultEndPort, + res: make(map[int]*Res), + } + + // apply opts + for _, o := range opts { + err = o(p) + if err != nil { + return nil, err + } + } + + return p, err +} + +// Shutdown releases all the pool resources. +func (p *Pool) Shutdown() error { + p.rw.Lock() + defer p.rw.Unlock() + + for _, r := range p.res { + r.cancel() + } + + return nil +} + +// Allocate creates a new process runner and returns it. +func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption) (*Res, error) { + var err error + + ctxt, cancel := context.WithCancel(ctxt) + + r := &Res{ + p: p, + ctxt: ctxt, + cancel: cancel, + port: p.next(), + } + + // create runner + r.r, err = runner.New(append([]runner.CommandLineOption{ + runner.Headless("", r.port), + }, opts...)...) + if err != nil { + cancel() + return nil, err + } + + // start runner + err = r.r.Start(ctxt) + if err != nil { + cancel() + return nil, err + } + + // setup cdp + r.c, err = New(ctxt, WithRunner(r.r)) + if err != nil { + cancel() + return nil, err + } + + p.rw.Lock() + defer p.rw.Unlock() + + p.res[r.port] = r + + return r, nil +} + +// next returns the next available port number. +func (p *Pool) next() int { + p.rw.Lock() + defer p.rw.Unlock() + + var found bool + var i int + for i = p.start; i < p.end; i++ { + if _, ok := p.res[i]; !ok { + found = true + break + } + } + + if !found { + panic("no ports available") + } + + return i +} + +// Res is a pool resource. +type Res struct { + p *Pool + ctxt context.Context + cancel func() + port int + r *runner.Runner + c *CDP +} + +// Release releases the pool resource. +func (r *Res) Release() error { + r.cancel() + + r.p.rw.Lock() + defer r.p.rw.Unlock() + + delete(r.p.res, r.port) + + return nil +} + +// Port returns the allocated port for the pool resource. +func (r *Res) Port() int { + return r.port +} + +// URL returns a formatted URL for the pool resource. +func (r *Res) URL() string { + return fmt.Sprintf("http://localhost:%d/json", r.port) +} + +// CDP returns the actual CDP instance. +func (r *Res) CDP() *CDP { + return r.c +} + +// Run runs an action. +func (r *Res) Run(ctxt context.Context, a Action) error { + return r.c.Run(ctxt, a) +} + +// PoolOption is a pool option. +type PoolOption func(*Pool) error + +// PortRange is a pool option to set the port range to use. +func PortRange(start, end int) PoolOption { + return func(p *Pool) error { + p.start = start + p.end = end + return nil + } +} diff --git a/runner/runner.go b/runner/runner.go index 2fed934..cb53037 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -334,6 +334,26 @@ func Path(path string) CommandLineOption { } } +// Headless is the Chrome command line option to set the default settings for +// running the headless_shell executable. If path is empty, then an attempt +// will be made to find headless_shell on the path. +func Headless(path string, port int) CommandLineOption { + if path == "" { + path, _ = exec.LookPath("headless_shell") + } + + return func(m map[string]interface{}) error { + m["exec-path"] = path + m["remote-debugging-port"] = port + + if os.Getenv("CHROMEDP_NO_SANDBOX") != "" { + m["no-sandbox"] = true + } + + return nil + } +} + // ExecPath is a Chrome command line option to set the exec path. func ExecPath(path string) CommandLineOption { return Flag("exec-path", path) @@ -370,6 +390,11 @@ func UserAgent(userAgent string) CommandLineOption { return Flag("user-agent", userAgent) } +// NoSandbox is the Chrome comamnd line option to disable the sandbox. +func NoSandbox(m map[string]interface{}) error { + return Flag("no-sandbox", true)(m) +} + // CmdOpt is a Chrome command line option to modify the underlying exec.Cmd // prior to invocation. func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {