diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f3614dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +/click/click +/cookie/cookie +/eval/eval +/keys/keys +/logic/logic +/pool/pool +/screenshot/screenshot +/simple/simple +/standalone/standalone +/submit/submit +/text/text +/upload/upload +/visible/visible + +*.png +*.exe diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8e2e6c2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2017 Kenneth Shaw + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b072be6 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# About chromedp examples + +This folder contains a variety of code examples for working with +[`chromedp`][1]. Please note that when the `chromedp` package is being +rewritten, these examples may break. Additionally, since these examples are +written for specific websites, there is a good chance that the current +selectors, etc break after the website they are written against changes. + +While every effort is made to ensure that these examples are kept up-to-date, +it is expected that the examples made available here +and working, As this is not a core part of the `chromedp` project, + +## Building and Running an Example + +You can build and run these examples in the usual Go way: + +```sh +# retrieve examples +$ go get -u -d github.com/chromedp/examples + +# run example +$ go run $GOPATH/src/github.com/chromedp/examples//main.go + +# build example +$ go build -o github.com/chromedp/examples/ && ./ +``` +### Available Examples + +The following examples are currently available: + + + +| Example | Description | +|---------------------------|-------------------------------------------------------------------------------------------| +| [click](/click) | use a selector to click on an element. | +| [cookie](/cookie) | set a HTTP cookie on requests. | +| [eval](/eval) | evaluate javascript and retrieve the result. | +| [keys](/keys) | send key events to an element. | +| [logic](/logic) | more complex logic beyond simple actions. | +| [pool](/pool) | use chromedp pool. | +| [screenshot](/screenshot) | take a screenshot of a specific element. | +| [simple](/simple) | do a simple google search. | +| [standalone](/standalone) | use chromedp with a standalone (ie, a "pre-existing" / manually started) chrome instance. | +| [submit](/submit) | fill out and submit a form. | +| [text](/text) | extract text from a specific element. | +| [upload](/upload) | upload a file on a form. | +| [visible](/visible) | wait until an element is visible. | + + +## Contributing + +Pull Requests and contributions to this project are encouraged and greatly +welcomed! The `chromedp` project always needs new examples, and needs talented +developers (such as yourself!) to submit fixes for the existing examples when +they break (for example, when a website's layout/HTML changes). + +[1]: https://github.com/chromedp/chromedp diff --git a/click/.gitignore b/click/.gitignore deleted file mode 100644 index bf3ee3d..0000000 --- a/click/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -click -click.exe diff --git a/click/main.go b/click/main.go index 02f7926..c8eded6 100644 --- a/click/main.go +++ b/click/main.go @@ -1,3 +1,5 @@ +// Command click is a chromedp example demonstrating how to use a selector to +// click on an element. package main import ( diff --git a/cookie/.gitignore b/cookie/.gitignore deleted file mode 100644 index 2cb56f0..0000000 --- a/cookie/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -cookie -cookie.exe diff --git a/cookie/main.go b/cookie/main.go index c9a1376..764e713 100644 --- a/cookie/main.go +++ b/cookie/main.go @@ -1,3 +1,5 @@ +// Command cookie is a chromedp example demonstrating how to set a HTTP cookie +// on requests. package main import ( diff --git a/eval/.gitignore b/eval/.gitignore deleted file mode 100644 index 13772a9..0000000 --- a/eval/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -eval -eval.exe diff --git a/eval/main.go b/eval/main.go index bf7f124..8963ccd 100644 --- a/eval/main.go +++ b/eval/main.go @@ -1,3 +1,5 @@ +// Command eval is a chromedp example demonstrating how to evaluate javascript +// and retrieve the result. package main import ( diff --git a/gen.go b/gen.go new file mode 100644 index 0000000..dbdbf17 --- /dev/null +++ b/gen.go @@ -0,0 +1,109 @@ +// +build ignore + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/parser" + "go/token" + "io/ioutil" + "log" + "path/filepath" + "regexp" + "sort" + "strings" +) + +const ( + sectionStart = "" + sectionEnd = "" + + descStart = "demonstrating" + descPrefix = "how to" +) + +var ( + flagReadme = flag.String("readme", "README.md", "file to update") + flagMask = flag.String("mask", "*/main.go", "") + + spaceRE = regexp.MustCompile(`\s+`) +) + +func main() { + flag.Parse() + + buf, err := ioutil.ReadFile(*flagReadme) + if err != nil { + log.Fatal(err) + } + + start, end := bytes.Index(buf, []byte(sectionStart)), bytes.Index(buf, []byte(sectionEnd)) + if start == -1 || end == -1 { + log.Fatalf("could not find %s or %s in %s", sectionStart, sectionEnd, *flagReadme) + } + + files, err := filepath.Glob(*flagMask) + if err != nil { + log.Fatal(err) + } + + type ex struct { + name, desc string + } + var examples []ex + for _, fn := range files { + f, err := parser.ParseFile(token.NewFileSet(), fn, nil, parser.ParseComments) + if err != nil { + log.Fatal(err) + } + + n := filepath.Base(filepath.Dir(fn)) + name := fmt.Sprintf("[%s](/%s)", n, n) + + // clean comment + comment := strings.Replace(f.Doc.Text(), "\n", " ", -1) + i := strings.Index(comment, descStart) + if i == -1 { + log.Fatalf("could not find %q in doc comment for %s", descStart, fn) + } + comment = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(comment[i+len(descStart):]), descPrefix)) + + examples = append(examples, ex{name, comment}) + } + sort.Slice(examples, func(i, j int) bool { return strings.Compare(examples[i].name, examples[j].name) < 0 }) + + // determine max length + var namelen, desclen int + for _, e := range examples { + namelen, desclen = max(namelen, len(e.name)), max(desclen, len(e.desc)) + } + + // generate + out := new(bytes.Buffer) + out.Write(buf[:start+len(sectionStart)]) + out.WriteString(fmt.Sprintf("\n| %s | %s |\n", pad("Example", " ", namelen), pad("Description", " ", desclen))) + out.WriteString(fmt.Sprintf("|%s|%s|\n", pad("", "-", namelen+2), pad("", "-", desclen+2))) + for _, e := range examples { + out.WriteString(fmt.Sprintf("| %s | %s |\n", pad(e.name, " ", namelen), pad(e.desc, " ", desclen))) + } + out.Write(buf[end:]) + + // write + err = ioutil.WriteFile(*flagReadme, out.Bytes(), 0644) + if err != nil { + log.Fatal(err) + } +} + +func max(a, b int) int { + if a >= b { + return a + } + return b +} + +func pad(s, v string, n int) string { + return s + strings.Repeat(v, n-len(s)) +} diff --git a/headless/.gitignore b/headless/.gitignore deleted file mode 100644 index 34160ad..0000000 --- a/headless/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -headless -headless.exe -*.png diff --git a/headless/README.md b/headless/README.md deleted file mode 100644 index d1581fc..0000000 --- a/headless/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# About headless - -This is a version of the simple example but with the chromedp settings changed -to use the docker [knqz/chrome-headless](https://hub.docker.com/r/knqz/chrome-headless/) image. - -## Running - -```sh -# retrieve docker image -docker pull knqz/chrome-headless - -# start chrome-headless -docker run -d -p 9222:9222 --rm --name chrome-headless knqz/chrome-headless - -# run chromedp headless example -go build && ./headless -``` diff --git a/keys/.gitignore b/keys/.gitignore deleted file mode 100644 index c142c01..0000000 --- a/keys/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -keys -keys.exe diff --git a/keys/main.go b/keys/main.go index 6e7e5ad..ffa84a2 100644 --- a/keys/main.go +++ b/keys/main.go @@ -1,3 +1,5 @@ +// Command keys is a chromedp example demonstrating how to send key events to +// an element. package main import ( diff --git a/logic/.gitignore b/logic/.gitignore deleted file mode 100644 index 2b06ea5..0000000 --- a/logic/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -logic -logic.exe diff --git a/logic/main.go b/logic/main.go index a8a4edc..0023106 100644 --- a/logic/main.go +++ b/logic/main.go @@ -1,4 +1,5 @@ -// examples/logic/main.go +// Command logic is a chromedp example demonstrating more complex logic beyond +// simple actions. package main import ( diff --git a/pool/.gitignore b/pool/.gitignore deleted file mode 100644 index adb2941..0000000 --- a/pool/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -pool -pool.exe - -*.png diff --git a/pool/main.go b/pool/main.go index 7a3e2e1..d485685 100644 --- a/pool/main.go +++ b/pool/main.go @@ -1,3 +1,4 @@ +// Command pool is a chromedp example demonstrating how to use chromedp pool. package main import ( diff --git a/screenshot/.gitignore b/screenshot/.gitignore deleted file mode 100644 index 7bb1d53..0000000 --- a/screenshot/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.png -screenshot -screenshot.exe diff --git a/screenshot/main.go b/screenshot/main.go index 53fd090..13475ad 100644 --- a/screenshot/main.go +++ b/screenshot/main.go @@ -1,3 +1,5 @@ +// Command screenshot is a chromedp example demonstrating how to take a +// screenshot of a specific element. package main import ( diff --git a/simple/.gitignore b/simple/.gitignore deleted file mode 100644 index 5e6880d..0000000 --- a/simple/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.png -simple -simple.exe diff --git a/simple/main.go b/simple/main.go index 86150b9..908a12b 100644 --- a/simple/main.go +++ b/simple/main.go @@ -1,4 +1,5 @@ -// examples/simple/main.go +// Command simple is a chromedp example demonstrating how to do a simple google +// search. package main import ( diff --git a/standalone/README.md b/standalone/README.md new file mode 100644 index 0000000..2deeac7 --- /dev/null +++ b/standalone/README.md @@ -0,0 +1,47 @@ +# About example standalone + +This is a version of the [`simple`][1] example but uses the `chromedp.WithTargets` +option with `chromedp/client.WatchPageTargets` to connect to an existing Chrome +instance (ie, no process will be launched). + +## Manually Starting Chrome + +To use this example, please manually start a Chrome instance with the `--remote-debugging-port=9222` +command-line option make the Chrome Debugging Protocol available to clients: + +```sh +# start google-chrome +$ google-chrome --remote-debugging-port=9222 + +# start headless_shell +$ headless_shell --headless --remote-debugging-port=9222 + +# start google-chrome-unstable in headless mode +$ google-chrome-unstable --headless --remote-debugging-port=9222 +``` + +### Docker Image + +A Docker image, [knqz/chrome-headless][2], provides a small ready-to-use +`headless_shell` that can be used with this example: + +```sh +# retrieve docker image +$ docker pull knqz/chrome-headless + +# start chrome-headless +$ docker run -d -p 9222:9222 --rm --name chrome-headless knqz/chrome-headless +``` + +[1]: ../simple +[2]: https://hub.docker.com/r/knqz/chrome-headless/ + +## Building and Running + +The `standalone` example can be run like any other Go code: + +```sh +# run example +$ cd $GOPATH/src/github.com/chromedp/examples/standalone +$ go build && ./standalone +``` diff --git a/headless/main.go b/standalone/main.go similarity index 90% rename from headless/main.go rename to standalone/main.go index d86e7db..63bd6a3 100644 --- a/headless/main.go +++ b/standalone/main.go @@ -1,3 +1,5 @@ +// Command standalone is a chromedp example demonstrating how to use chromedp +// with a standalone (ie, a "pre-existing" / manually started) chrome instance. package main import ( diff --git a/submit/.gitignore b/submit/.gitignore deleted file mode 100644 index 415822e..0000000 --- a/submit/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -submit -submit.exe diff --git a/submit/main.go b/submit/main.go index 242d7be..4ae3253 100644 --- a/submit/main.go +++ b/submit/main.go @@ -1,3 +1,5 @@ +// Command submit is a chromedp example demonstrating how to fill out and +// submit a form. package main import ( diff --git a/text/.gitignore b/text/.gitignore deleted file mode 100644 index de10394..0000000 --- a/text/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -text -text.exe diff --git a/text/main.go b/text/main.go index 6c1168e..94c41f3 100644 --- a/text/main.go +++ b/text/main.go @@ -1,3 +1,5 @@ +// Command text is a chromedp example demonstrating how to extract text from a +// specific element. package main import ( diff --git a/upload/.gitignore b/upload/.gitignore deleted file mode 100644 index a0f3426..0000000 --- a/upload/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -upload -upload.exe diff --git a/upload/main.go b/upload/main.go index 3e0bc44..4f7641b 100644 --- a/upload/main.go +++ b/upload/main.go @@ -1,3 +1,5 @@ +// Command upload is a chromedp example demonstrating how to upload a file on a +// form. package main import ( diff --git a/visible/.gitignore b/visible/.gitignore deleted file mode 100644 index 1c9969e..0000000 --- a/visible/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -visible -visible.exe diff --git a/visible/main.go b/visible/main.go index f8e3424..a9fd08d 100644 --- a/visible/main.go +++ b/visible/main.go @@ -1,3 +1,5 @@ +// Command visible is a chromedp example demonstrating how to wait until an +// element is visible. package main import (