2018-04-01 09:03:21 +00:00
|
|
|
package browsh
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
2022-06-27 16:54:50 +00:00
|
|
|
"embed"
|
2018-04-01 09:03:21 +00:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
2023-12-09 09:14:52 +00:00
|
|
|
"io/fs"
|
2023-12-07 16:32:00 +00:00
|
|
|
"log/slog"
|
2018-04-01 09:03:21 +00:00
|
|
|
"net"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
2022-06-27 16:54:50 +00:00
|
|
|
"os/signal"
|
2019-06-19 13:54:32 +00:00
|
|
|
"path"
|
2018-07-11 07:43:34 +00:00
|
|
|
"regexp"
|
2018-06-10 12:16:28 +00:00
|
|
|
"runtime"
|
2018-04-01 09:03:21 +00:00
|
|
|
"strings"
|
2022-06-27 16:54:50 +00:00
|
|
|
"syscall"
|
2018-04-01 09:03:21 +00:00
|
|
|
"time"
|
|
|
|
|
2018-05-27 12:31:47 +00:00
|
|
|
"github.com/gdamore/tcell"
|
2018-04-01 09:03:21 +00:00
|
|
|
"github.com/go-errors/errors"
|
2018-07-17 10:43:52 +00:00
|
|
|
"github.com/spf13/viper"
|
2018-04-01 09:03:21 +00:00
|
|
|
)
|
|
|
|
|
2022-06-27 16:54:50 +00:00
|
|
|
//go:embed browsh.xpi
|
|
|
|
var browshXpi embed.FS
|
|
|
|
|
2018-04-28 04:07:39 +00:00
|
|
|
var (
|
|
|
|
marionette net.Conn
|
|
|
|
ffCommandCount = 0
|
|
|
|
defaultFFPrefs = map[string]string{
|
|
|
|
"startup.homepage_welcome_url.additional": "''",
|
|
|
|
"devtools.errorconsole.enabled": "true",
|
|
|
|
"devtools.chrome.enabled": "true",
|
|
|
|
|
|
|
|
// Send Browser Console (different from Devtools console) output to
|
|
|
|
// STDOUT.
|
|
|
|
"browser.dom.window.dump.enabled": "true",
|
|
|
|
|
|
|
|
// From:
|
|
|
|
// http://hg.mozilla.org/mozilla-central/file/1dd81c324ac7/build/automation.py.in//l388
|
|
|
|
// Make url-classifier updates so rare that they won"t affect tests.
|
|
|
|
"urlclassifier.updateinterval": "172800",
|
|
|
|
// Point the url-classifier to a nonexistent local URL for fast failures.
|
|
|
|
"browser.safebrowsing.provider.0.gethashURL": "'http://localhost/safebrowsing-dummy/gethash'",
|
|
|
|
"browser.safebrowsing.provider.0.keyURL": "'http://localhost/safebrowsing-dummy/newkey'",
|
|
|
|
"browser.safebrowsing.provider.0.updateURL": "'http://localhost/safebrowsing-dummy/update'",
|
|
|
|
|
|
|
|
// Disable self repair/SHIELD
|
|
|
|
"browser.selfsupport.url": "'https://localhost/selfrepair'",
|
|
|
|
// Disable Reader Mode UI tour
|
|
|
|
"browser.reader.detectedFirstArticle": "true",
|
|
|
|
|
|
|
|
// Set the policy firstURL to an empty string to prevent
|
|
|
|
// the privacy info page to be opened on every "web-ext run".
|
|
|
|
// (See #1114 for rationale)
|
|
|
|
"datareporting.policy.firstRunURL": "''",
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
2018-04-01 09:03:21 +00:00
|
|
|
func startHeadlessFirefox() {
|
2023-12-07 16:32:00 +00:00
|
|
|
slog.Info("Starting Firefox in headless mode")
|
2019-06-10 17:43:27 +00:00
|
|
|
checkIfFirefoxIsAlreadyRunning()
|
2018-07-19 04:27:38 +00:00
|
|
|
firefoxPath := ensureFirefoxBinary()
|
|
|
|
ensureFirefoxVersion(firefoxPath)
|
2018-04-01 09:03:21 +00:00
|
|
|
args := []string{"--marionette"}
|
2018-07-17 10:43:52 +00:00
|
|
|
if !viper.GetBool("firefox.with-gui") {
|
2018-04-01 09:03:21 +00:00
|
|
|
args = append(args, "--headless")
|
|
|
|
}
|
2018-07-17 10:43:52 +00:00
|
|
|
profile := viper.GetString("firefox.profile")
|
2018-07-18 11:52:22 +00:00
|
|
|
if profile != "browsh-default" {
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("Using Firefox profile", "profile", profile)
|
2018-07-17 10:43:52 +00:00
|
|
|
args = append(args, "-P", profile)
|
2018-04-01 09:03:21 +00:00
|
|
|
} else {
|
2018-07-17 10:43:52 +00:00
|
|
|
profilePath := getFirefoxProfilePath()
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("Using default profile", "path", profilePath)
|
2018-04-01 09:03:21 +00:00
|
|
|
args = append(args, "--profile", profilePath)
|
|
|
|
}
|
2018-07-19 04:27:38 +00:00
|
|
|
firefoxProcess := exec.Command(firefoxPath, args...)
|
2018-04-01 09:03:21 +00:00
|
|
|
defer firefoxProcess.Process.Kill()
|
|
|
|
stdout, err := firefoxProcess.StdoutPipe()
|
|
|
|
if err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
|
|
|
if err := firefoxProcess.Start(); err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
|
|
|
in := bufio.NewScanner(stdout)
|
|
|
|
for in.Scan() {
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("FF-CONSOLE", "stdout", in.Text())
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-12 07:40:49 +00:00
|
|
|
func checkIfFirefoxIsAlreadyRunning() {
|
2018-07-11 07:43:34 +00:00
|
|
|
if runtime.GOOS == "windows" {
|
|
|
|
return
|
|
|
|
}
|
2018-06-12 07:40:49 +00:00
|
|
|
processes := Shell("ps aux")
|
|
|
|
r, _ := regexp.Compile("firefox.*--headless")
|
|
|
|
if r.MatchString(processes) {
|
|
|
|
Shutdown(errors.New("A headless Firefox is already running"))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-19 04:27:38 +00:00
|
|
|
func ensureFirefoxBinary() string {
|
2018-07-17 10:43:52 +00:00
|
|
|
path := viper.GetString("firefox.path")
|
|
|
|
if path == "firefox" {
|
2018-06-10 12:16:28 +00:00
|
|
|
switch runtime.GOOS {
|
|
|
|
case "windows":
|
2018-07-17 10:43:52 +00:00
|
|
|
path = getFirefoxPath()
|
2018-06-10 12:16:28 +00:00
|
|
|
case "darwin":
|
2018-07-17 10:43:52 +00:00
|
|
|
path = "/Applications/Firefox.app/Contents/MacOS/firefox"
|
2018-06-10 12:16:28 +00:00
|
|
|
default:
|
2018-07-17 10:43:52 +00:00
|
|
|
path = getFirefoxPath()
|
2018-06-10 12:16:28 +00:00
|
|
|
}
|
|
|
|
}
|
2023-12-09 09:14:52 +00:00
|
|
|
if _, err := os.Stat(path); err != nil {
|
|
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
|
|
err = errors.New("Firefox binary not found: " + path)
|
|
|
|
}
|
|
|
|
Shutdown(err)
|
2018-07-11 07:43:34 +00:00
|
|
|
}
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("Using Firefox", "path", path)
|
2018-07-19 04:27:38 +00:00
|
|
|
return path
|
2018-07-11 07:43:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Taken from https://stackoverflow.com/a/18411978/575773
|
|
|
|
func versionOrdinal(version string) string {
|
|
|
|
// ISO/IEC 14651:2011
|
|
|
|
const maxByte = 1<<8 - 1
|
|
|
|
vo := make([]byte, 0, len(version)+8)
|
|
|
|
j := -1
|
|
|
|
for i := 0; i < len(version); i++ {
|
|
|
|
b := version[i]
|
|
|
|
if '0' > b || b > '9' {
|
|
|
|
vo = append(vo, b)
|
|
|
|
j = -1
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if j == -1 {
|
|
|
|
vo = append(vo, 0x00)
|
|
|
|
j = len(vo) - 1
|
|
|
|
}
|
|
|
|
if vo[j] == 1 && vo[j+1] == '0' {
|
|
|
|
vo[j+1] = b
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if vo[j]+1 > maxByte {
|
|
|
|
panic("VersionOrdinal: invalid version")
|
|
|
|
}
|
|
|
|
vo = append(vo, b)
|
|
|
|
vo[j]++
|
|
|
|
}
|
|
|
|
return string(vo)
|
|
|
|
}
|
|
|
|
|
2018-04-01 09:03:21 +00:00
|
|
|
// Start Firefox via the `web-ext` CLI tool. This is for development and testing,
|
|
|
|
// because I haven't been able to recreate the way `web-ext` injects an unsigned
|
|
|
|
// extension.
|
|
|
|
func startWERFirefox() {
|
2023-12-07 16:32:00 +00:00
|
|
|
slog.Info("Attempting to start headless Firefox with `web-ext`")
|
2019-06-10 17:43:27 +00:00
|
|
|
if IsConnectedToWebExtension {
|
|
|
|
Shutdown(errors.New("There appears to already be an existing Web Extension connection"))
|
|
|
|
}
|
|
|
|
checkIfFirefoxIsAlreadyRunning()
|
2023-12-07 16:32:00 +00:00
|
|
|
rootDir := Shell("git rev-parse --show-toplevel")
|
2018-04-01 09:03:21 +00:00
|
|
|
args := []string{
|
|
|
|
"run",
|
|
|
|
"--firefox=" + rootDir + "/webext/contrib/firefoxheadless.sh",
|
|
|
|
"--verbose",
|
|
|
|
"--no-reload",
|
|
|
|
}
|
2018-07-11 07:43:34 +00:00
|
|
|
firefoxProcess := exec.Command(rootDir+"/webext/node_modules/.bin/web-ext", args...)
|
2018-04-01 09:03:21 +00:00
|
|
|
firefoxProcess.Dir = rootDir + "/webext/dist/"
|
|
|
|
stdout, err := firefoxProcess.StdoutPipe()
|
|
|
|
if err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
|
|
|
if err := firefoxProcess.Start(); err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
|
|
|
in := bufio.NewScanner(stdout)
|
|
|
|
for in.Scan() {
|
2019-06-10 17:43:27 +00:00
|
|
|
if strings.Contains(in.Text(), "Connected to the remote Firefox debugger") {
|
|
|
|
}
|
2018-04-01 09:03:21 +00:00
|
|
|
if strings.Contains(in.Text(), "JavaScript strict") ||
|
2018-07-11 07:43:34 +00:00
|
|
|
strings.Contains(in.Text(), "D-BUS") ||
|
|
|
|
strings.Contains(in.Text(), "dbus") {
|
2018-04-01 09:03:21 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("FF-CONSOLE", "stdout", in.Text())
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
2023-12-07 16:32:00 +00:00
|
|
|
slog.Info("WER Firefox unexpectedly closed")
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Connect to Firefox's Marionette service.
|
|
|
|
// RANT: Firefox's remote control tools are so confusing. There seem to be 2
|
|
|
|
// services that come with your Firefox binary; Marionette and the Remote
|
|
|
|
// Debugger. The latter you would expect to follow the widely supported
|
|
|
|
// Chrome standard, but no, it's merely on the roadmap. There is very little
|
|
|
|
// documentation on either. I have the impression, but I'm not sure why, that
|
|
|
|
// the Remote Debugger is better, seemingly more API methods, and as mentioned
|
|
|
|
// is on the roadmap to follow the Chrome standard.
|
|
|
|
// I've used Marionette here, simply because it was easier to reverse engineer
|
|
|
|
// from the Python Marionette package.
|
|
|
|
func firefoxMarionette() {
|
2018-06-12 11:38:39 +00:00
|
|
|
var (
|
2018-07-11 07:43:34 +00:00
|
|
|
err error
|
2018-06-12 11:38:39 +00:00
|
|
|
conn net.Conn
|
|
|
|
)
|
|
|
|
connected := false
|
2023-12-07 16:32:00 +00:00
|
|
|
slog.Info("Attempting to connect to Firefox Marionette")
|
2018-06-12 11:38:39 +00:00
|
|
|
start := time.Now()
|
2018-07-11 07:43:34 +00:00
|
|
|
for time.Since(start) < 30*time.Second {
|
2018-06-12 11:38:39 +00:00
|
|
|
conn, err = net.Dial("tcp", "127.0.0.1:2828")
|
|
|
|
if err != nil {
|
2018-07-16 14:29:10 +00:00
|
|
|
if !strings.Contains(err.Error(), "refused") {
|
2018-06-12 11:38:39 +00:00
|
|
|
Shutdown(err)
|
|
|
|
} else {
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
connected = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !connected {
|
|
|
|
Shutdown(errors.New("Failed to connect to Firefox's Marionette within 30 seconds"))
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
marionette = conn
|
2019-06-18 16:05:08 +00:00
|
|
|
go readMarionette()
|
2018-11-06 09:06:07 +00:00
|
|
|
sendFirefoxCommand("WebDriver:NewSession", map[string]interface{}{})
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func installWebextension() {
|
2022-06-27 16:54:50 +00:00
|
|
|
data, err := browshXpi.ReadFile("browsh.xpi")
|
2018-04-01 09:03:21 +00:00
|
|
|
if err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
2019-06-19 13:54:32 +00:00
|
|
|
path := path.Join(os.TempDir(), "browsh-webext-addon")
|
2023-12-09 09:14:52 +00:00
|
|
|
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
|
|
|
|
Shutdown(err)
|
|
|
|
}
|
2019-06-19 13:54:32 +00:00
|
|
|
args := map[string]interface{}{"path": path}
|
2018-11-06 09:06:07 +00:00
|
|
|
sendFirefoxCommand("Addon:Install", args)
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Set a Firefox preference as you would in `about:config`
|
|
|
|
// `value` needs to be supplied with quotes if it's to be used as a JS string
|
|
|
|
func setFFPreference(key string, value string) {
|
2019-06-18 13:38:06 +00:00
|
|
|
var args map[string]interface{}
|
|
|
|
var script string
|
2018-11-06 09:06:07 +00:00
|
|
|
sendFirefoxCommand("Marionette:SetContext", map[string]interface{}{"value": "chrome"})
|
2019-06-18 13:38:06 +00:00
|
|
|
script = fmt.Sprintf(`
|
2018-04-01 09:03:21 +00:00
|
|
|
Components.utils.import("resource://gre/modules/Preferences.jsm");
|
2019-06-18 13:38:06 +00:00
|
|
|
prefs = new Preferences({defaultBranch: "root"});
|
|
|
|
prefs.set("%s", %s);`, key, value)
|
|
|
|
args = map[string]interface{}{"script": script}
|
2018-11-06 09:06:07 +00:00
|
|
|
sendFirefoxCommand("WebDriver:ExecuteScript", args)
|
|
|
|
sendFirefoxCommand("Marionette:SetContext", map[string]interface{}{"value": "content"})
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Consume output from Marionette, we don't do anything with it. It"s just
|
|
|
|
// useful to have it in the logs.
|
|
|
|
func readMarionette() {
|
|
|
|
buffer := make([]byte, 4096)
|
|
|
|
count, err := marionette.Read(buffer)
|
|
|
|
if err != nil {
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Error("Error reading from Marionette connection", "error", err)
|
2019-06-18 16:05:08 +00:00
|
|
|
return
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("FF-MRNT", "buffer", string(buffer[:count]))
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func sendFirefoxCommand(command string, args map[string]interface{}) {
|
2023-12-09 09:14:52 +00:00
|
|
|
slog.Info("Sending command to Firefox Marionette", "command", command, "args", args)
|
2018-04-01 09:03:21 +00:00
|
|
|
fullCommand := []interface{}{0, ffCommandCount, command, args}
|
|
|
|
marshalled, _ := json.Marshal(fullCommand)
|
|
|
|
message := fmt.Sprintf("%d:%s", len(marshalled), marshalled)
|
|
|
|
fmt.Fprintf(marionette, message)
|
|
|
|
ffCommandCount++
|
2019-06-18 16:05:08 +00:00
|
|
|
go readMarionette()
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
2019-06-18 13:38:06 +00:00
|
|
|
func setDefaultFirefoxPreferences() {
|
2018-04-01 09:03:21 +00:00
|
|
|
for key, value := range defaultFFPrefs {
|
|
|
|
setFFPreference(key, value)
|
|
|
|
}
|
2019-06-18 16:05:08 +00:00
|
|
|
for _, pref := range viper.GetStringSlice("firefox.preferences") {
|
|
|
|
parts := strings.SplitN(pref, "=", 2)
|
|
|
|
setFFPreference(parts[0], parts[1])
|
2019-06-18 13:38:06 +00:00
|
|
|
}
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func beginTimeLimit() {
|
|
|
|
warningLength := 10
|
2018-07-11 07:43:34 +00:00
|
|
|
warningLimit := time.Duration(*timeLimit - warningLength)
|
2018-04-01 09:03:21 +00:00
|
|
|
time.Sleep(warningLimit * time.Second)
|
|
|
|
message := fmt.Sprintf("Browsh will close in %d seconds...", warningLength)
|
|
|
|
sendMessageToWebExtension("/status," + message)
|
|
|
|
time.Sleep(time.Duration(warningLength) * time.Second)
|
2018-06-23 09:16:12 +00:00
|
|
|
quitBrowsh()
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|
|
|
|
|
2019-06-18 13:38:06 +00:00
|
|
|
// Careful what you change here as it isn't tested during CI
|
2018-04-01 09:03:21 +00:00
|
|
|
func setupFirefox() {
|
|
|
|
go startHeadlessFirefox()
|
2018-07-11 07:43:34 +00:00
|
|
|
if *timeLimit > 0 {
|
2018-04-01 09:03:21 +00:00
|
|
|
go beginTimeLimit()
|
|
|
|
}
|
2022-06-27 16:54:50 +00:00
|
|
|
sigs := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
go func() {
|
|
|
|
<-sigs
|
|
|
|
quitBrowsh()
|
2022-07-16 18:02:37 +00:00
|
|
|
}()
|
2022-06-27 16:54:50 +00:00
|
|
|
|
2018-04-01 09:03:21 +00:00
|
|
|
firefoxMarionette()
|
|
|
|
installWebextension()
|
2018-06-12 07:40:49 +00:00
|
|
|
}
|
|
|
|
|
2019-06-10 17:43:27 +00:00
|
|
|
func StartFirefox() {
|
2018-07-17 10:43:52 +00:00
|
|
|
if !viper.GetBool("firefox.use-existing") {
|
2018-07-06 05:06:49 +00:00
|
|
|
writeString(0, 16, "Waiting for Firefox to connect...", tcell.StyleDefault)
|
2018-05-27 12:31:47 +00:00
|
|
|
if IsTesting {
|
2018-07-06 05:06:49 +00:00
|
|
|
writeString(0, 17, "TEST MODE", tcell.StyleDefault)
|
2018-05-27 12:31:47 +00:00
|
|
|
go startWERFirefox()
|
2019-06-18 13:38:06 +00:00
|
|
|
firefoxMarionette()
|
2018-05-27 12:31:47 +00:00
|
|
|
} else {
|
|
|
|
setupFirefox()
|
|
|
|
}
|
|
|
|
} else {
|
2019-06-18 13:38:06 +00:00
|
|
|
firefoxMarionette()
|
2018-07-06 05:06:49 +00:00
|
|
|
writeString(0, 16, "Waiting for a user-initiated Firefox instance to connect...", tcell.StyleDefault)
|
2018-05-27 12:31:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-01 09:03:21 +00:00
|
|
|
func quitFirefox() {
|
2018-11-06 09:06:07 +00:00
|
|
|
sendFirefoxCommand("Marionette:Quit", map[string]interface{}{})
|
2018-04-01 09:03:21 +00:00
|
|
|
}
|