Adding logic example
This commit is contained in:
parent
28ee6338d2
commit
457ad699ce
2
examples/logic/.gitignore
vendored
Normal file
2
examples/logic/.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
logic
|
||||||
|
logic.exe
|
107
examples/logic/main.go
Normal file
107
examples/logic/main.go
Normal file
|
@ -0,0 +1,107 @@
|
||||||
|
// examples/logic/main.go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
cdp "github.com/knq/chromedp"
|
||||||
|
cdptypes "github.com/knq/chromedp/cdp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
// create context
|
||||||
|
ctxt, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// create chrome instance
|
||||||
|
c, err := cdp.New(ctxt, cdp.WithLog(log.Printf))
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// list awesome go projects for the "Selenium and browser control tools."
|
||||||
|
res, err := listAwesomeGoProjects(ctxt, c, "Selenium and browser control tools.")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("could not list awesome go projects: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shutdown chrome
|
||||||
|
err = c.Shutdown(ctxt)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait for chrome to finish
|
||||||
|
err = c.Wait()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// output the values
|
||||||
|
for k, v := range res {
|
||||||
|
log.Printf("project %s (%s): '%s'", k, v.URL, v.Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ud contains a url, description for a project.
|
||||||
|
type ud struct {
|
||||||
|
URL, Description string
|
||||||
|
}
|
||||||
|
|
||||||
|
// listAwesomeGoProjects is the highest level logic for browsing to the
|
||||||
|
// awesome-go page, finding the specified section sect, and retrieving the
|
||||||
|
// associated projects from the page.
|
||||||
|
func listAwesomeGoProjects(ctxt context.Context, c *cdp.CDP, sect string) (map[string]ud, error) {
|
||||||
|
// force max timeout of 15 seconds for retrieving and processing the data
|
||||||
|
var cancel func()
|
||||||
|
ctxt, cancel = context.WithTimeout(ctxt, 25*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
sel := fmt.Sprintf(`//p[text()[contains(., '%s')]]`, sect)
|
||||||
|
|
||||||
|
// navigate
|
||||||
|
if err := c.Run(ctxt, cdp.Navigate(`https://github.com/avelino/awesome-go`)); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not navigate to github: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait visible
|
||||||
|
if err := c.Run(ctxt, cdp.WaitVisible(sel)); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not get section: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sib := sel + `/following-sibling::ul/li`
|
||||||
|
|
||||||
|
// get project link text
|
||||||
|
var projects []*cdptypes.Node
|
||||||
|
if err := c.Run(ctxt, cdp.Nodes(sib+`/child::a/text()`, &projects)); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not get projects: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// get links and description text
|
||||||
|
var linksAndDescriptions []*cdptypes.Node
|
||||||
|
if err := c.Run(ctxt, cdp.Nodes(sib+`/child::node()`, &linksAndDescriptions)); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not get links and descriptions: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check length
|
||||||
|
if 2*len(projects) != len(linksAndDescriptions) {
|
||||||
|
return nil, fmt.Errorf("projects and links and descriptions lengths do not match (2*%d != %d)", len(projects), len(linksAndDescriptions))
|
||||||
|
}
|
||||||
|
|
||||||
|
// process data
|
||||||
|
res := make(map[string]ud)
|
||||||
|
for i := 0; i < len(projects); i++ {
|
||||||
|
res[projects[i].NodeValue] = ud{
|
||||||
|
URL: linksAndDescriptions[2*i].AttributeValue("href"),
|
||||||
|
Description: strings.TrimPrefix(strings.TrimSpace(linksAndDescriptions[2*i+1].NodeValue), "- "),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user