Skip to content

Commit

Permalink
Merge branch 'downloader'
Browse files Browse the repository at this point in the history
  • Loading branch information
matthewmueller committed Dec 3, 2017
2 parents dfba36d + 14f64de commit 106943b
Show file tree
Hide file tree
Showing 4 changed files with 231 additions and 4 deletions.
13 changes: 12 additions & 1 deletion cmd/golly/golly.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"strings"
"syscall"

"github.com/matthewmueller/golly/internal/chrome"
"github.com/matthewmueller/golly/internal/compiler/util"

kingpin "gopkg.in/alecthomas/kingpin.v2"
Expand Down Expand Up @@ -81,8 +82,18 @@ func run(ctx context.Context) error {
}
filePath := path.Join(cwd, *runFile)

root, err := util.GollyPath()
if err != nil {
return errors.Wrapf(err, "error getting root path")
}

chromePath, err := chrome.Find(path.Join(root, "chrome"))
if err != nil {
return errors.Wrapf(err, "unable to get chrome path")
}

result, err := api.Run(ctx, &api.RunSettings{
ChromePath: os.Getenv("GOLLY_CHROME_PATH"),
ChromePath: chromePath,
FilePath: filePath,
})
if err != nil {
Expand Down
10 changes: 10 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
const puppeteer = require('puppeteer');

(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://news.ycombinator.com', { waitUntil: 'networkidle2' });
await page.pdf({ path: 'hn.pdf', format: 'A4' });

await browser.close();
})();
37 changes: 34 additions & 3 deletions internal/chrome/chrome.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/url"
"os"
"os/exec"
"path"
"strconv"
"strings"
"syscall"
Expand Down Expand Up @@ -40,6 +41,31 @@ type Settings struct {
Addr string
}

var defaultFlags = []string{
"--disable-background-networking",
"--disable-background-timer-throttling",
"--disable-client-side-phishing-detection",
"--disable-default-apps",
"--disable-extensions",
"--disable-hang-monitor",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--no-first-run",
"--safebrowsing-disable-auto-update",

"--enable-automation",
"--password-store=basic",
"--use-mock-keychain",

"--headless",
"--disable-gpu",
"--hide-scrollbars",
"--mute-audio",
}

var errStopped = errors.New("stopped")

// New chrome
Expand All @@ -58,11 +84,16 @@ func New(parent context.Context, settings *Settings) (*Chrome, error) {
// default flags
flags := append(
settings.Flags,
"--headless",
"--disable-gpu",
"--remote-debugging-port="+addr.Port(),
defaultFlags...,
)

// user data dir
tmp := os.TempDir()
flags = append(flags, "--user-data-dir="+path.Join(tmp))

// remote debugging port
flags = append(flags, "--remote-debugging-port="+addr.Port())

// create the command
cmd := exec.CommandContext(ctx, settings.ExecutablePath, flags...)

Expand Down
175 changes: 175 additions & 0 deletions internal/chrome/downloader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
package chrome

import (
"archive/zip"
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"runtime"

"github.com/pkg/errors"
)

// ChromiumRevision chromium revision
const ChromiumRevision = "518818"

// DownloadHost download host
const DownloadHost = "https://storage.googleapis.com"

var downloadURLs = map[string]string{
"linux": "%s/chromium-browser-snapshots/Linux_x64/%s/chrome-linux.zip",
"mac": "%s/chromium-browser-snapshots/Mac/%s/chrome-mac.zip",
"win32": "%s/chromium-browser-snapshots/Win/%s/chrome-win32.zip",
"win64": "%s/chromium-browser-snapshots/Win_x64/%s/chrome-win32.zip",
}

// Find locally or remotely
func Find(dir string) (string, error) {
filepath, err := chromiumPath(dir)
if err != nil {
return "", errors.Wrapf(err, "error getting chromium path")
}

if _, err := os.Stat(filepath); !os.IsNotExist(err) {
return filepath, nil
}

if err := download(dir); err != nil {
return "", errors.Wrapf(err, "error downloading")
}

return filepath, nil
}

// Download path
func download(dir string) error {
var url, platform string

switch runtime.GOOS {
case "darwin":
platform = "mac"
url = fmt.Sprintf(downloadURLs[platform], DownloadHost, ChromiumRevision)
case "linux":
platform = "linux"
url = fmt.Sprintf(downloadURLs[platform], DownloadHost, ChromiumRevision)
case "windows":
platform = "windows"
// TODO: 32 vs 64
url = fmt.Sprintf(downloadURLs["win64"], DownloadHost, ChromiumRevision)
default:
return errors.New("unsupported operating system")
}

res, err := http.Get(url)
if err != nil {
return errors.Wrapf(err, "error getting download")
}
defer res.Body.Close()

src, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}

folderPath := path.Join(dir, platform+"-"+ChromiumRevision)
if err := os.MkdirAll(folderPath, 0775); err != nil {
return errors.Wrapf(err, "error making directory")
}

return unzip(src, folderPath)
}

// Path to the executable
// TODO: test on windows
func chromiumPath(dir string) (string, error) {
var platform string
switch runtime.GOOS {
case "darwin":
platform = "mac"
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"), nil
case "linux":
platform = "linux"
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-linux", "chrome"), nil
case "windows":
platform = "windows"
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-win32", "chrome.exe"), nil
default:
return "", errors.New("unsupported operating system")
}
}

// Unzip a file to a destination and preserve the symlinks
// Based on http://stackoverflow.com/a/24792688/842097 with symlink additions
func unzip(src []byte, dest string) error {
rdr := bytes.NewReader(src)

r, err := zip.NewReader(rdr, rdr.Size())
if err != nil {
return err
}

os.MkdirAll(dest, 0755)

// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
panic(err)
}
}()

path := filepath.Join(dest, f.Name)

if f.FileInfo().IsDir() {
os.MkdirAll(path, f.Mode())
} else if f.FileInfo().Mode()&os.ModeSymlink != 0 {
buffer := make([]byte, f.FileInfo().Size())
size, err := rc.Read(buffer)
if err != nil {
return err
}

target := string(buffer[:size])

err = os.Symlink(target, path)
if err != nil {
return err
}
} else {
os.MkdirAll(filepath.Dir(path), f.Mode())
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err = f.Close(); err != nil {
panic(err)
}
}()

_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}

for _, f := range r.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}

return nil
}

0 comments on commit 106943b

Please sign in to comment.