Fixing issues with unit tests

1. Changes travis configuration to use the chrome addon
2. Adds more command line options to runner package
3. Removes grab-headless_shell.sh script
4. Cleans up and adds some environment variables for controlling how
   unit tests are ran
5. Fixes a minor issue in chromedp-gen on comment output (for working
   with latest protocol definition)

.travis.yml

@@ -5,14 +5,16 @@ go:
   - tip
 addons:
   apt:
+    chrome: stable
     sources:
       - ubuntu-toolchain-r-test
     packages:
       - libstdc++6
 before_install:
   - go get github.com/mattn/goveralls
-  - ./contrib/grab-headless_shell.sh $HOME/hs
 script:
   - export PATH=$PATH:$HOME/hs
-  - CHROMEDP_NO_SANDBOX=true go test -v -coverprofile=coverage.out
+  - export CHROMEDP_TEST_RUNNER=google-chrome-stable
+  - export CHROMEDP_DISABLE_GPU=true
+  - go test -v -coverprofile=coverage.out
   - goveralls -service=travis-ci -coverprofile=coverage.out

chromedp.go

@@ -83,7 +83,7 @@ func New(ctxt context.Context, opts ...Option) (*CDP, error) {
 	}()
 
 	// TODO: fix this
-	timeout := time.After(DefaultNewTargetTimeout)
+	timeout := time.After(defaultNewTargetTimeout)
 
 loop:
 	// wait until at least one target active
@@ -430,3 +430,9 @@ func WithConsolef(f LogFunc) Option {
 		return nil
 	}
 }
+
+var (
+	// defaultNewTargetTimeout is the default target timeout -- used by
+	// testing.
+	defaultNewTargetTimeout = DefaultNewTargetTimeout
+)

chromedp_test.go

@@ -5,14 +5,25 @@ import (
 	"log"
 	"os"
 	"testing"
+	"time"
+
+	"github.com/knq/chromedp/runner"
 )
 
-var pool *Pool
+var (
-var defaultContext, defaultCancel = context.WithCancel(context.Background())
+	pool        *Pool
-var testdataDir string
+	testdataDir string
+
+	defaultContext, defaultCancel = context.WithCancel(context.Background())
+
+	cliOpts = []runner.CommandLineOption{
+		runner.NoDefaultBrowserCheck,
+		runner.NoFirstRun,
+	}
+)
 
 func testAllocate(t *testing.T, path string) *Res {
-	c, err := pool.Allocate(defaultContext)
+	c, err := pool.Allocate(defaultContext, cliOpts...)
 	if err != nil {
 		t.Fatalf("could not allocate from pool: %v", err)
 	}
@@ -59,6 +70,27 @@ func TestMain(m *testing.M) {
 
 	testdataDir = "file://" + os.Getenv("GOPATH") + "/src/github.com/knq/chromedp/testdata"
 
+	// its worth noting that newer versions of chrome (64+) run much faster
+	// than older ones -- same for headless_shell ...
+	if testRunner := os.Getenv("CHROMEDP_TEST_RUNNER"); testRunner != "" {
+		cliOpts = append(cliOpts, runner.ExecPath(testRunner))
+	}
+	// not explicitly needed to be set, as this vastly speeds up unit tests
+	if noSandbox := os.Getenv("CHROMEDP_NO_SANDBOX"); noSandbox != "false" {
+		cliOpts = append(cliOpts, runner.NoSandbox)
+	}
+	// must be explicitly set, as disabling gpu slows unit tests
+	if disableGPU := os.Getenv("CHROMEDP_DISABLE_GPU"); disableGPU != "" && disableGPU != "false" {
+		cliOpts = append(cliOpts, runner.DisableGPU)
+	}
+
+	if targetTimeout := os.Getenv("CHROMEDP_TARGET_TIMEOUT"); targetTimeout != "" {
+		defaultNewTargetTimeout, _ = time.ParseDuration(targetTimeout)
+	}
+	if defaultNewTargetTimeout == 0 {
+		defaultNewTargetTimeout = 30 * time.Second
+	}
+
 	//pool, err = NewPool(PoolLog(log.Printf, log.Printf, log.Printf))
 	pool, err = NewPool()
 	if err != nil {

cmd/chromedp-gen/internal/util.go

@@ -27,10 +27,12 @@ func init() {
 	misspellReplacer.Compile()
 }
 
-var badHTMLReplacer = strings.NewReplacer(
+var descReplacer = strings.NewReplacer(
 	"&lt;", "<",
 	"&gt;", ">",
 	"&gt", ">",
+	"`", "",
+	"\n", " ",
 )
 
 // codeRE is a regexp to match <code> and </code> tags.
@@ -39,8 +41,8 @@ var codeRE = regexp.MustCompile(`<\/?code>`)
 // CleanDesc cleans comments / descriptions of "<code>" and "</code>" strings
 // and "`" characters, and fixes common misspellings.
 func CleanDesc(s string) string {
-	s, _ = misspellReplacer.Replace(strings.Replace(codeRE.ReplaceAllString(s, ""), "`", "", -1))
+	s, _ = misspellReplacer.Replace(codeRE.ReplaceAllString(s, ""))
-	return badHTMLReplacer.Replace(s)
+	return descReplacer.Replace(s)
 }
 
 // ForceCamel forces camel case specific to go.

contrib/grab-headless_shell.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-OUT=${1:-headless_shell}
-VER=$2
-
-if [ -z "$VER" ]; then
-  VER=$(curl -s https://storage.googleapis.com/docker-chrome-headless/latest.txt|sed -e 's/^headless_shell-//' -e 's/\.tar\.bz2$//')
-fi
-
-mkdir -p $OUT
-
-pushd $OUT &> /dev/null
-
-curl -s https://storage.googleapis.com/docker-chrome-headless/headless_shell-$VER.tar.bz2 | tar -jxv
-
-./headless_shell --remote-debugging-port=8222 &
-
-HEADLESS_PID=$!
-
-sleep 1
-
-curl -v -q http://localhost:8222/json/version
-
-kill -9 $HEADLESS_PID
-
-popd &> /dev/null

pool.go

@@ -72,7 +72,7 @@ func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption)
 
 	// create runner
 	r.r, err = runner.New(append([]runner.CommandLineOption{
-		runner.Headless("", r.port),
+		runner.HeadlessPathPort("", r.port),
 	}, opts...)...)
 	if err != nil {
 		defer r.Release()

runner/runner.go

@@ -336,10 +336,10 @@ func Path(path string) CommandLineOption {
 	}
 }
 
-// Headless is the Chrome command line option to set the default settings for
+// HeadlessPathPort is the Chrome command line option to set the default
-// running the headless_shell executable. If path is empty, then an attempt
+// settings for running the headless_shell executable. If path is empty, then
-// will be made to find headless_shell on the path.
+// an attempt will be made to find headless_shell on the path.
-func Headless(path string, port int) CommandLineOption {
+func HeadlessPathPort(path string, port int) CommandLineOption {
 	if path == "" {
 		path, _ = exec.LookPath("headless_shell")
 	}
@@ -347,11 +347,7 @@ func Headless(path string, port int) CommandLineOption {
 	return func(m map[string]interface{}) error {
 		m["exec-path"] = path
 		m["remote-debugging-port"] = port
-
+		m["headless"] = true
-		if os.Getenv("CHROMEDP_NO_SANDBOX") != "" {
-			m["no-sandbox"] = true
-		}
-
 		return nil
 	}
 }
@@ -367,6 +363,10 @@ func Port(port int) CommandLineOption {
 }
 
 // UserDataDir is the Chrome command line option to set the user data dir.
+//
+// Note: set this option to manually set the profile directory used by Chrome.
+// When this is not set, then a default path will be created in the /tmp
+// directory.
 func UserDataDir(dir string) CommandLineOption {
 	return Flag("user-data-dir", dir)
 }
@@ -402,6 +402,23 @@ func NoSandbox(m map[string]interface{}) error {
 	return Flag("no-sandbox", true)(m)
 }
 
+// NoFirstRun is the Chrome comamnd line option to disable the first run
+// dialog.
+func NoFirstRun(m map[string]interface{}) error {
+	return Flag("no-first-run", true)(m)
+}
+
+// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the
+// default browser check.
+func NoDefaultBrowserCheck(m map[string]interface{}) error {
+	return Flag("no-default-browser-check", true)(m)
+}
+
+// DisableGPU is the Chrome command line option to disable the GPU process.
+func DisableGPU(m map[string]interface{}) error {
+	return Flag("disable-gpu", true)(m)
+}
+
 // CmdOpt is a Chrome command line option to modify the underlying exec.Cmd
 // prior to invocation.
 func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {