From 148e24a615f1641312e96b3c6993b53057006a5a Mon Sep 17 00:00:00 2001 From: Kenneth Shaw Date: Mon, 18 Dec 2017 07:11:42 +0700 Subject: [PATCH] Fixing issues with unit tests 1. Changes travis configuration to use the chrome addon 2. Adds more command line options to runner package 3. Removes grab-headless_shell.sh script 4. Cleans up and adds some environment variables for controlling how unit tests are ran 5. Fixes a minor issue in chromedp-gen on comment output (for working with latest protocol definition) --- .travis.yml | 6 +++-- chromedp.go | 8 ++++++- chromedp_test.go | 40 +++++++++++++++++++++++++++---- cmd/chromedp-gen/internal/util.go | 8 ++++--- contrib/grab-headless_shell.sh | 28 ---------------------- pool.go | 2 +- runner/runner.go | 35 ++++++++++++++++++++------- 7 files changed, 79 insertions(+), 48 deletions(-) delete mode 100755 contrib/grab-headless_shell.sh diff --git a/.travis.yml b/.travis.yml index bed478c..d9abc80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,16 @@ go: - tip addons: apt: + chrome: stable sources: - ubuntu-toolchain-r-test packages: - libstdc++6 before_install: - go get github.com/mattn/goveralls - - ./contrib/grab-headless_shell.sh $HOME/hs script: - export PATH=$PATH:$HOME/hs - - CHROMEDP_NO_SANDBOX=true go test -v -coverprofile=coverage.out + - export CHROMEDP_TEST_RUNNER=google-chrome-stable + - export CHROMEDP_DISABLE_GPU=true + - go test -v -coverprofile=coverage.out - goveralls -service=travis-ci -coverprofile=coverage.out diff --git a/chromedp.go b/chromedp.go index 98b5f80..fe12985 100644 --- a/chromedp.go +++ b/chromedp.go @@ -83,7 +83,7 @@ func New(ctxt context.Context, opts ...Option) (*CDP, error) { }() // TODO: fix this - timeout := time.After(DefaultNewTargetTimeout) + timeout := time.After(defaultNewTargetTimeout) loop: // wait until at least one target active @@ -430,3 +430,9 @@ func WithConsolef(f LogFunc) Option { return nil } } + +var ( + // defaultNewTargetTimeout is the default target timeout -- used by + // testing. + defaultNewTargetTimeout = DefaultNewTargetTimeout +) diff --git a/chromedp_test.go b/chromedp_test.go index fce39ec..679f72b 100644 --- a/chromedp_test.go +++ b/chromedp_test.go @@ -5,14 +5,25 @@ import ( "log" "os" "testing" + "time" + + "github.com/knq/chromedp/runner" ) -var pool *Pool -var defaultContext, defaultCancel = context.WithCancel(context.Background()) -var testdataDir string +var ( + pool *Pool + testdataDir string + + defaultContext, defaultCancel = context.WithCancel(context.Background()) + + cliOpts = []runner.CommandLineOption{ + runner.NoDefaultBrowserCheck, + runner.NoFirstRun, + } +) func testAllocate(t *testing.T, path string) *Res { - c, err := pool.Allocate(defaultContext) + c, err := pool.Allocate(defaultContext, cliOpts...) if err != nil { t.Fatalf("could not allocate from pool: %v", err) } @@ -59,6 +70,27 @@ func TestMain(m *testing.M) { testdataDir = "file://" + os.Getenv("GOPATH") + "/src/github.com/knq/chromedp/testdata" + // its worth noting that newer versions of chrome (64+) run much faster + // than older ones -- same for headless_shell ... + if testRunner := os.Getenv("CHROMEDP_TEST_RUNNER"); testRunner != "" { + cliOpts = append(cliOpts, runner.ExecPath(testRunner)) + } + // not explicitly needed to be set, as this vastly speeds up unit tests + if noSandbox := os.Getenv("CHROMEDP_NO_SANDBOX"); noSandbox != "false" { + cliOpts = append(cliOpts, runner.NoSandbox) + } + // must be explicitly set, as disabling gpu slows unit tests + if disableGPU := os.Getenv("CHROMEDP_DISABLE_GPU"); disableGPU != "" && disableGPU != "false" { + cliOpts = append(cliOpts, runner.DisableGPU) + } + + if targetTimeout := os.Getenv("CHROMEDP_TARGET_TIMEOUT"); targetTimeout != "" { + defaultNewTargetTimeout, _ = time.ParseDuration(targetTimeout) + } + if defaultNewTargetTimeout == 0 { + defaultNewTargetTimeout = 30 * time.Second + } + //pool, err = NewPool(PoolLog(log.Printf, log.Printf, log.Printf)) pool, err = NewPool() if err != nil { diff --git a/cmd/chromedp-gen/internal/util.go b/cmd/chromedp-gen/internal/util.go index 83ef345..6c5f266 100644 --- a/cmd/chromedp-gen/internal/util.go +++ b/cmd/chromedp-gen/internal/util.go @@ -27,10 +27,12 @@ func init() { misspellReplacer.Compile() } -var badHTMLReplacer = strings.NewReplacer( +var descReplacer = strings.NewReplacer( "<", "<", ">", ">", ">", ">", + "`", "", + "\n", " ", ) // codeRE is a regexp to match and tags. @@ -39,8 +41,8 @@ var codeRE = regexp.MustCompile(`<\/?code>`) // CleanDesc cleans comments / descriptions of "" and "" strings // and "`" characters, and fixes common misspellings. func CleanDesc(s string) string { - s, _ = misspellReplacer.Replace(strings.Replace(codeRE.ReplaceAllString(s, ""), "`", "", -1)) - return badHTMLReplacer.Replace(s) + s, _ = misspellReplacer.Replace(codeRE.ReplaceAllString(s, "")) + return descReplacer.Replace(s) } // ForceCamel forces camel case specific to go. diff --git a/contrib/grab-headless_shell.sh b/contrib/grab-headless_shell.sh deleted file mode 100755 index 2a21286..0000000 --- a/contrib/grab-headless_shell.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -ex - -OUT=${1:-headless_shell} -VER=$2 - -if [ -z "$VER" ]; then - VER=$(curl -s https://storage.googleapis.com/docker-chrome-headless/latest.txt|sed -e 's/^headless_shell-//' -e 's/\.tar\.bz2$//') -fi - -mkdir -p $OUT - -pushd $OUT &> /dev/null - -curl -s https://storage.googleapis.com/docker-chrome-headless/headless_shell-$VER.tar.bz2 | tar -jxv - -./headless_shell --remote-debugging-port=8222 & - -HEADLESS_PID=$! - -sleep 1 - -curl -v -q http://localhost:8222/json/version - -kill -9 $HEADLESS_PID - -popd &> /dev/null diff --git a/pool.go b/pool.go index 5a0dc63..5e2fe75 100644 --- a/pool.go +++ b/pool.go @@ -72,7 +72,7 @@ func (p *Pool) Allocate(ctxt context.Context, opts ...runner.CommandLineOption) // create runner r.r, err = runner.New(append([]runner.CommandLineOption{ - runner.Headless("", r.port), + runner.HeadlessPathPort("", r.port), }, opts...)...) if err != nil { defer r.Release() diff --git a/runner/runner.go b/runner/runner.go index 8d1a7a9..1f38cff 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -336,10 +336,10 @@ func Path(path string) CommandLineOption { } } -// Headless is the Chrome command line option to set the default settings for -// running the headless_shell executable. If path is empty, then an attempt -// will be made to find headless_shell on the path. -func Headless(path string, port int) CommandLineOption { +// HeadlessPathPort is the Chrome command line option to set the default +// settings for running the headless_shell executable. If path is empty, then +// an attempt will be made to find headless_shell on the path. +func HeadlessPathPort(path string, port int) CommandLineOption { if path == "" { path, _ = exec.LookPath("headless_shell") } @@ -347,11 +347,7 @@ func Headless(path string, port int) CommandLineOption { return func(m map[string]interface{}) error { m["exec-path"] = path m["remote-debugging-port"] = port - - if os.Getenv("CHROMEDP_NO_SANDBOX") != "" { - m["no-sandbox"] = true - } - + m["headless"] = true return nil } } @@ -367,6 +363,10 @@ func Port(port int) CommandLineOption { } // UserDataDir is the Chrome command line option to set the user data dir. +// +// Note: set this option to manually set the profile directory used by Chrome. +// When this is not set, then a default path will be created in the /tmp +// directory. func UserDataDir(dir string) CommandLineOption { return Flag("user-data-dir", dir) } @@ -402,6 +402,23 @@ func NoSandbox(m map[string]interface{}) error { return Flag("no-sandbox", true)(m) } +// NoFirstRun is the Chrome comamnd line option to disable the first run +// dialog. +func NoFirstRun(m map[string]interface{}) error { + return Flag("no-first-run", true)(m) +} + +// NoDefaultBrowserCheck is the Chrome comamnd line option to disable the +// default browser check. +func NoDefaultBrowserCheck(m map[string]interface{}) error { + return Flag("no-default-browser-check", true)(m) +} + +// DisableGPU is the Chrome command line option to disable the GPU process. +func DisableGPU(m map[string]interface{}) error { + return Flag("disable-gpu", true)(m) +} + // CmdOpt is a Chrome command line option to modify the underlying exec.Cmd // prior to invocation. func CmdOpt(o func(*exec.Cmd) error) CommandLineOption {