Enterprise-Onion-Toolkit/eotk

#!/bin/sh
# Enterprise Onion Toolkit

# A NOTE TO CONTRIBUTORS: yes this is a big script, but it's also
# written to be easy to understand (and reason about) what it is
# doing; this is why there are `for` loops around `scp` rather than
# tarballing up a bunch of file and shipping them to remote machines,
# unpacking them there. This is (mostly) why there are some
# long_variable_names. I plan to keep it that way.

#rsync_flags="-n" # testing

# expected by tools and libraries
cd `dirname $0` || exit 1
export EOTK_HOME=`pwd`

# for invoking myself
prog=`basename $0`
self=$EOTK_HOME/$prog

# meta
version=1.2

# set project directory; this path is hard-replicated elsewhere
project_dir=$EOTK_HOME/projects.d

# mirror directory
mirrors_dir=$EOTK_HOME/mirrors.d

# onionbalance directory
ob_dir=$EOTK_HOME/onionbalance.d
ob_conf=$ob_dir/config.yaml
ob_status_sock=$ob_dir/ob-status.sock
ob_tor_conf=$ob_dir/tor.conf
ob_tor_control_sock=$ob_dir/tor-control.sock

# init script name
init=eotk-init.sh

# log retention
log_retain=8

# set path
export PATH=$EOTK_HOME/opt.d:$EOTK_HOME/lib.d:$EOTK_HOME:$PATH

##################################################################

# argument stripping
flag_remote=false
flag_local=false
flag_boot=false
propagate_flags=""
while : ; do
    case "x$1" in
        x--remote)
            flag_remote=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--local)
            flag_local=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--boot) # `smart` --local, + override some PID checks
            flag_boot=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--debug)
            set -x
            shift
            ;;
        *)
            break
            ;;
    esac
done

# what are the hostnames of the remote workers?
host_list=$EOTK_HOME/eotk-workers.conf

CloudHosts() {
    # use --remote hack to forcibly stop risk of recursion...
    if $flag_remote ; then
        echo "localhost" # this will be treated as a magic sentinel
        return 0
    fi

    # perversely, --local, exactly the same
    if $flag_local ; then
        echo "localhost"
        return 0
    fi

    # if --boot, we want localhost IFF we would have it anyway...
    if $flag_boot ; then
        if [ -s $host_list ] ; then # does it contain localhost...?
            egrep '^(localhost)\b' $host_list
        else
            echo "localhost"
        fi
        return 0
    fi

    # todo: if eotk-hosts.conf exists,
    # cat it (strip comments?)
    # and return
    if [ -s $host_list ] ; then
        cat $host_list
        return 0
    fi

    # else we are working just on this machine
    echo "localhost" # this will be treated as a magic sentinel
}

cloud_hosts=`CloudHosts` # saves multiple invocations
need_to_run_locally=false
for host in $cloud_hosts ; do
    test "x$host" = "xlocalhost" && need_to_run_locally=true
done

# delete dangerous stuff on boot
OnBootCleanup() {
    pidfiles=`find $EOTK_HOME -type f -name "*.pid"`
    sockfiles=`find $EOTK_HOME -type s -name "*.sock"`
    for file in $pidfiles $sockfiles ; do
        if [ -e $file ] ; then # -e => "exists"
            rm $file || exit 1
        fi
        Warn purged $file
    done
}

##################################################################

# print a formatted message to stdout
Print() {
    echo "$prog:" "$@"
}

# print a formatted message to stderr
Warn() {
    Print "$@" 1>&2
}

# compress logfiles
LogRotate() {
    SUFFIX=.bz2
    do_proj=$1 # true|false
    do_ob=$2 # true|false
    WHERE=""
    $do_proj && WHERE="$WHERE $project_dir"
    $do_ob && WHERE="$WHERE $ob_dir"
    if [ "x$WHERE" = "x" ] ; then
        Print error: LogRotate has no targets
        exit 1
    fi
    logfiles=`find $WHERE -type f -name "*.log" -print`
    for logfile in $logfiles ; do
        hi=`expr $log_retain - 1`
        dst=$logfile.$hi$SUFFIX
        if [ -f $dst ] ; then
            rm $dst || exit 1
            Print purged $dst
        fi
        while [ $hi -gt 0 ] ; do
            lo=`expr $hi - 1`
            src=$logfile.$lo$SUFFIX
            dst=$logfile.$hi$SUFFIX
            if [ -f $src ] ; then
                mv $src $dst || exit 1
                Print bumped $dst
            fi
            hi=$lo
        done
        dst=$logfile.$hi
        mv $logfile $dst || exit 1
        Print created $dst
        to_compress="$to_compress $dst"
    done
    if $do_proj ; then
        $self --local nxreload -a
        $self --local torreload -a
    fi
    if $do_ob ; then
        $self ob-restart -a
    fi
    bzip2 -fv $to_compress
}

# essentially the projects.d folder is a little database
ListProjects() {
    if [ x$1 = x--softmap-only ] ; then
        softmap_only=true
    else
        softmap_only=false
    fi
    (
        test -d $project_dir || exit 1
        cd $project_dir || exit 1
        for d in *.d/ ; do
            if $softmap_only ; then
                test -f $d/softmap.conf || continue
            fi
            echo `basename $d .d`
        done
    )
}

# TODO(alecm) resolve potential clashes between project names and
# various other directory names ending in ".d"; maybe use ".proj"?

SpotPush() {
    for filename in "$@" ; do
        files=`find $project_dir -type f -name "$filename" | sort`
        for host in $cloud_hosts ; do
            test "x$host" = "xlocalhost" && continue # skip self
            echo :::: push $host ::::
            for filepath in $files ; do
                Print copying $filepath to $host
                scp -p $filepath $host:$filepath || exit 1
            done
        done
    done
}

# push eotk directory to remote
DestructivePush() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: rnap $host ::::
        rsync $rsync_flags \
              -av \
              --delete \
              --delete-excluded \
              --exclude="*.log" \
              --exclude="*.pid" \
              --exclude="*.sock" \
              --exclude="*.yaml" \
              --exclude="*~" \
              --exclude="docs.d/" \
              --exclude="mirrors.d/" \
              --exclude="onionbalance.d/" \
              --exclude="secrets.d/" \
              --exclude=".git/" \
              --exclude=".gitignore" \
              --exclude="cached-certs" \
              --exclude="cached-microdesc*" \
              --exclude="configure*.log" \
              --exclude="eotk-workers.conf" \
              --exclude="hostname" \
              --exclude="lock" \
              --exclude="onion_service_non_anonymous" \
              --exclude="private_key" \
              --exclude="state" \
              ${EOTK_HOME}/ \
              $host:${EOTK_HOME}/
    done
}

# mirror remote back for log review, backup, etc
Mirror() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: mirror $host ::::
        test -d $mirrors_dir || mkdir -p $mirrors_dir || exit 1
        chmod 700 $mirrors_dir || exit 1
        rsync $rsync_flags -av \
              --delete \
              --delete-excluded \
              --exclude="cached-certs" \
              --exclude="cached-microdesc*" \
              $host:${EOTK_HOME}/ $mirrors_dir/$host/
    done
}

# run a command in the context of the local projects directory
RunLocally() {
    action=$1
    shift
    project=$1
    shift

    echo :::: $action $project $@ ::::
    sh "$project_dir/$project.d/$action.sh" "$@"
}

# $1=action, rest = project names
RunLocallyOverProjects() {
    action=$1
    shift # remaining arguments are projects

    if [ "x$1" = "x" ] ; then # test for no args
        Print error: missing project name, try: $prog projects for a list, or -a for all
        return 1
    elif [ "x$1" = "x-a" ] ; then # test for / expand the "-a" flag
        projects=`ListProjects`
    else # do what we are told
        projects="$*"
    fi

    # loop
    for project in $projects ; do
        RunLocally $action $project
    done
}

# run a command on remote machines, or possibly locally
InvokeRemotely() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: remote $host: $* ::::
        ssh "$host" "$self --remote $*"
    done
}

# get a config file and re/populate the projects directory with it
Configure() {
    log=configure$$.log

    for file in "$@" ; do
        echo :::: configure $file ::::

        case "$file" in
            *.conf)
                : happy bunnies
                ;;

            *.tconf)
                file2=`basename $file .tconf`.conf
                if [ -s $file2 ] ; then
                    Print info: $file: using existing $file2
                else
                    Print info: Processing $file
                    Print info: Populating $file2 with onions
                    Print info: Please be patient...
                    expand-config.pl $file >$file2
                fi
                file="$file2"
                ;;

            *)
                Print error: bad config file suffix, was expecting: .conf, .tconf
                exit 1
                ;;
        esac

        if ! $EOTK_HOME/lib.d/do-configure.pl "$file" ; then
            Print error: failure processing $file: see $log
            exit 1
        fi
    done 2>$log

    Print done. logfile is $log.
}

# argument 'parser' - ha!

cmd="$1" # we may need the remaining args
shift

case "$cmd" in
    version|test)
        test -d .git && git show -s --oneline
        tor --version || Warn cannot find tor executable
        nginx -v || Warn cannot find nginx executable
        onionbalance --version || Warn cannot find onionbalance executable
        Print $version $EOTK_HOME `uname -a`
        InvokeRemotely version
        ;;

    projects|proj)
        ListProjects
        ;;

    populate|pop)
        Populate "$@"
        ;;

    configure|config|conf)
        Configure "$@"
        ;;

    genkey|gen)
        secrets_dir=secrets.d
        test -d $secrets_dir || mkdir -p $secrets_dir || exit 1
        chmod 700 $secrets_dir || exit 1
        (
            cd $secrets_dir
            generate-onion-key.sh
        )
        ;;

    ## ACTIONS

    start) # project ..., or "-a"
        $flag_boot && OnBootCleanup
        $need_to_run_locally && RunLocallyOverProjects start "$@"
        InvokeRemotely start "$@"
        ;;

    stop) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects stop "$@"
        InvokeRemotely stop "$@"
        ;;

    restart|bounce|reload) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects bounce "$@"
        InvokeRemotely bounce "$@"
        ;;

    nxreload|nx-reload) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects nxreload "$@"
        InvokeRemotely nxreload "$@"
        ;;

    torreload|tor-reload) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects torreload "$@"
        InvokeRemotely torreload "$@"
        ;;

    debugon|debug-on) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects debugon "$@"
        InvokeRemotely debugon "$@"
        ;;

    debugoff|debug-off) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects debugoff "$@"
        InvokeRemotely debugoff "$@"
        ;;

    syntax) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects syntax "$@"
        InvokeRemotely syntax "$@"
        ;;

    harvest|onions) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects harvest "$@"
        InvokeRemotely harvest "$@"
        ;;

    status) # project ..., or "-a"
        $self ob-ps # because common
        $need_to_run_locally && RunLocallyOverProjects status "$@"
        InvokeRemotely status "$@"
        ;;

    maps|map) # project ..., or "-a"
        $need_to_run_locally && RunLocallyOverProjects maps "$@"
        InvokeRemotely maps "$@"
        ;;

    delete) # project ..., or "-a"
        Print $cmd not yet implemented, sorry.
        ;;

    ## DIAGS

    ps)
        $flag_remote || echo :::: eotk processes ::::
        ps auxww | egrep '\b(eotk|onionbalance|nginx)\b'
        # InvokeRemotely prints remote-ness banner diags
        InvokeRemotely ps
        ;;

    ## ---- ONIONBALANCE ----

    push)
        Print push is destructive and has been renamed, see the documentation
        ;;

    shutdown)
        $self $propagate_flags ob-stop
        $self $propagate_flags stop -a
        ;;

    # this used to be called 'push' but got renamed because oops.
    # ---- DO NOT USE THIS CASUALLY, LEARN FROM MY MISTAKES ----
    ob-remote-nuke-and-push|rnap)
        $self stop -a
        DestructivePush
        ;;

    ob-nxpush)
        # gently push only the nginx configs to the workers
        SpotPush nginx.conf
        ;;

    ob-torpush)
        # gently push only the tor configs to the workers
        SpotPush tor.conf
        ;;

    ob-spotpush)
        # gently push only the tor configs to the workers
        SpotPush "$@"
        ;;

    ob-config|obconfig|ob-configure|obconfigure)
        Print ob-configure had been rolled into ob-start
        Print please skip forward to: $prog ob-start project ...
        ;;

    ob-gather|obgather) # project ..., or "-a"
        # ob storage
        test -d $ob_dir || mkdir -p $ob_dir || exit 1
        chmod 700 $ob_dir # otherwise tor complains

        # sanity check that args are provided
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        elif [ "x$1" = "x-a" ] ; then # test for / expand the "-a" flag
            projects=`ListProjects --softmap-only`
        else # do what we are told
            projects="$*"
        fi

        tor_address=127.0.0.1
        tor_port=9055

        # get the mappings
        Print gathering mappings for OnionBalance for projects: $projects
        mappings=__om$$.tmp
        $self maps $projects |
            awk '/^::::/{next;} $2=="softmap"{print;}' >$mappings

        # check the mappings
        for p in $projects ; do
            n=`awk '$1=="'"$p"'"'<$mappings  | wc -l`
            if [ $n = 0 ] ; then
                Warn no mappings yet for project $p: does it exist / is it started / is it not a softmap project "?"
                rm $mappings
                exit 1
            else
                m=`awk '$1=="'"$p"'"{print $3}'<$mappings | sort -u | wc -l`
                w=`awk '$1=="'"$p"'"{print $6}'<$mappings | sort -u | wc -l`
                Print project $p contains $n mappings
                Print project $p uses $m master onions
                Print project $p uses $w worker onions
            fi
        done

        Print building OnionBalance configurations...
        (
            echo LOG_LEVEL: info
            echo TOR_ADDRESS: $tor_address
            echo TOR_PORT: $tor_port
            echo REFRESH_INTERVAL: 300 # non-default, expect fast cycling
            echo PUBLISH_CHECK_INTERVAL: 150 # non-default, 0.5x REFRESH_INTERVAL
            echo INITIAL_DELAY: 90 # non-default
            echo STATUS_SOCKET_LOCATION: $ob_status_sock
            echo DESCRIPTOR_VALIDITY_PERIOD: 86400 # 1 day
            echo DESCRIPTOR_OVERLAP_PERIOD: 3600 # 1 hour
            echo DESCRIPTOR_UPLOAD_PERIOD: 1200 # non-default, 3x per hour
            do-obconfig.pl <$mappings
        ) > $ob_conf

        # clean up
        rm $mappings

        Print building OnionBalance-Tor configurations...
        (
            echo DataDirectory $ob_dir
            echo ControlPort unix:$ob_tor_control_sock
            echo PidFile $ob_dir/tor.pid
            echo Log info file $ob_dir/tor.log
            echo SafeLogging 1
            echo HeartbeatPeriod 60 minutes
            echo RunAsDaemon 1
            echo "#" onionbalance
            # echo SocksPort unix:$ob_dir/tor-socks.sock # curl 7.38 does not like this
            echo SocksPort $tor_address:$tor_port
            echo CookieAuthentication 1
            # echo MaxClientCircuitsPending 1024
        ) > $ob_tor_conf
        ;;

    ob-start|obstart) # project ..., or "-a"
        # do not test for ob_dir here, it is okay to not exist

        if [ -f $ob_dir/ob.pid ] ; then
            Warn file $ob_dir/ob.pid already exists, OnionBalance may already be running.
            Warn Aborting...
            exit 1
        fi

        if [ -f $ob_dir/tor.pid ] ; then
            Warn file $ob_dir/tor.pid already exists, OnionBalance-Tor may already be running.
            Warn Aborting...
            exit 1
        fi

        # ob-gather checks args
        $self ob-gather "$@" || exit 1

        Print starting OnionBalance-Tor
        tor -f $ob_tor_conf >$ob_dir/tor-startup.log 2>&1

        Print starting OnionBalance
        onionbalance \
            -s $ob_tor_control_sock \
            -c $ob_conf </dev/null >$ob_dir/onionbalance.log 2>&1 & # bg
        ob_pid=$!
        echo $ob_pid >$ob_dir/ob.pid
        ;;

    ob-stop|obstop)
        for pidfile in $ob_dir/ob.pid $ob_dir/tor.pid ; do
            test -s $pidfile || continue
            pid=`cat $pidfile`
            Print sending SIGTERM to $pid in $pidfile
            kill $pid
        done
        rm -f $ob_dir/ob.pid
        ;;

    ob-restart) # project ..., or "-a" : TODO: should these propagate flags?
        # ob-stop takes no args, but ob-start requires them; try to be consistent, therefore:
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        fi
        $self ob-stop
        $self ob-start "$@"
        ;;

    ob-ps)
        test -d $ob_dir || exit 0
        echo :::: onionbalance processes ::::
        pidfiles=`find $ob_dir -name "*.pid"`
        if [ "x$pidfiles" != "x" ] ; then
            ps -p `cat $pidfiles`
        fi
        ;;

    ob-status|obstatus)
        $self ob-ps
        if [ -S $ob_status_sock ] ; then
            echo ""
            socat - unix-connect:$ob_status_sock
        fi
        ;;

    ob-maps|obmaps) # project ..., or "-a"
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        fi
        $self maps "$@" |
            awk '/^::::/{next;} $2=="softmap"{print $3, $4}' |
            sort -k 2 -u
        ;;

    ## FREEZE/BACKUP

    mirror|pull)
        Mirror
        ;;

    backup|freeze)
        Mirror
        (
            ds=`date "+%Y%m%d%H%M%S"`
            cd $mirrors_dir || exit 1
            for directory in */ ; do
                test -d "$directory" || exit 1 # did */ expand?
                dir=`basename $directory` # strip trailing /
                echo :::: backup $dir ::::
                tar cf - $dir | bzip2 > $dir-$ds.tar.bz2
            done
        )
        ;;

    logrotate)
        # compress and restart the remotes, first
        InvokeRemotely logrotate
        # do we need to do onionbalance logs?
        if [ -d $ob_dir ] ; then
            need_to_do_ob_logs=true
        else
            need_to_do_ob_logs=false
        fi
        LogRotate $need_to_run_locally $need_to_do_ob_logs
        ;;

    make-init-script)
        lib.d/expand-template.pl templates.d/$init.txt </dev/null >$init
        chmod 755 $init
        ;;

    *)
        Print usage: see README.md for documentation
        exit 1
        ;;
esac

exit 0