proxy_role/overlay/Linux/usr/local/bin/scurl.bash

#!/bin/bash
# -*- mode: sh; tab-width: 8; coding: utf-8-unix -*-

# must not use stdout

prog=$( basename $0 .bash )
PREFIX=/usr/local
ROLE=proxy
prog=scurl
umask 022

RETRIES=2
SSL_VER=3

. /usr/local/bin/proxy_ping_lib.bash
[ -f /usr/local/bin/proxy_curl_lib.bash ] && \
   . /usr/local/bin/proxy_curl_lib.bash
if ! grep -q "^wlan[1-9][ 	]00000000" /proc/net/route ; then
    WARN $prog we are not connected >&2
    exit -1
fi

usage="curls with some wget options

Usage: $prog options -- curl-options

Options:
 -P, --directory-prefix
 -X, --force-directories create directories to download to
 -C, --cacert            CA certs in .pem
 -M, --mode              proxy_ping_mode
 -S, --ssl               ssl version 2=tls1.2 3=tls1.3
 -Y, --ciphers           comma sep list of ciphers
 -Q, --quiet             --silent --show-error
 -h, --help              display this help and exit
"
if [[ $? -ne 0 ]]; then
    echo "$usage"
    exit 2
fi

declare -a LARGS
# --location is required to follow redirects
# im not sure about  --http2
LARGS+=( --remote-time --location --max-redirs 10 --continue-at - )
LARGS+=( --retry-delay 10 --show-error --fail )
# --proto-redir https --proto =https is required to prevent protocol downgrades
LARGS+=( --proto-redir https --proto-default https --proto =https )

[ -z "$MODE" ] && MODE=$( /usr/local/bin/proxy_ping_lib.bash proxy_ping_mode )
[ -z "$socks_proxy" ] && . /usr/local/bin/proxy_export.bash

SSL_LIB=openssl # nss
if [ -x $PREFIX/bin/curl.bash ] ; then
    EXE=$PREFIX/bin/curl.bash
elif which scurl ; then
    EXE=`which scurl`
else
    EXE=curl
fi

SHORT=M:QP:XC:F:hS:
LONG=mode:,quiet,directory-prefix:,force-directories,cacert,ca-cert:,help,ssl:

#? export POSIXLY_CORRECT=1

PARSED=$(getopt --options $SHORT --longoptions $LONG --name "$prog" -- "$@")
eval set -- "$PARSED"

P="$PWD"
X="0"
# echo DEBUG: WD=$WD rest=$*
LOGF=/tmp/$prog$$.err
SSL_CIPHERS=""

while true; do
    case "$1" in
        -P|--directory-prefix)
	    shift
            P="$1"
            shift
	    # echo DEBUG: P=$WD rest=$*
            ;;
        -X|--force-directories)
            X=1
            shift
            ;;
        -S|--ssl)
            shift
            SSL_VER="$1"
            shift
            ;;
        -Y|--ciphers)
            shift
            SSL_CIPHERS="$1"
            shift
            ;;
        -C|--cacert|-Z|--ca-cert)
            shift
            CA_CERT="$1"
            shift
            ;;
        -M|--mode)
            shift
            MODE="$1"
            shift
            ;;
        -Q|--quiet)
            shift
	    LARGS="$LARGS --silent  --show-error"
            ;;
        -h|--help)
            echo USAGE: "$usage"
            exit 0
            ;;
        --)
            shift
            break
            ;;
        *)
	    # echo ERROR: unhandled arguments $* - use -- after -P $PWD or -X ; exit 3
	    break
            ;;
    esac
done

[ "$SSL_VER" -ge 2 -a "$SSL_VER" -le 3 ] || { ERROR "SSL_VER $SSL_VER" ; exit 6 ; }
LARGS+=( --tlsv1.$SSL_VER )

if [ -n "$SSL_CIPHERS" -a "$SSL_VER" = 2 ] ; then
    [ $SSL_LIB = openssl ] && \
      SSL_CIPHERS="ECDHE-RSA-AES256-SHA" # ECDHE-RSA-AES256-GCM-SHA384
    [ $SSL_LIB = nss ] && \
	SSL_CIPHERS="ecdhe_rsa_aes_256_sha"
fi

if [ -n "$SSL_CIPHERS" -a "$SSL_VER" = 3 ] ; then
    [ $SSL_LIB = openssl ] && \
      SSL_CIPHERS="TLS_AES_256_GCM_SHA384" # TLS_CHACHA20_POLY1305_SHA256
    [ $SSL_LIB = nss ] && \
      SSL_CIPHERS="aes_256_gcm_sha_384"
fi
[ -n "$SSL_CIPHERS" ] && LARGS+=(  --ciphers "$SSL_CIPHERS" )

if [ "$MODE" = tor -o "$MODE" = selektor -o "$MODE" = whonix ] && \
       netstat -nle4 | grep -q 127.0.0.1:53 ; then
    LARGS+=( --dns-ipv4-addr 127.0.0.1 --dns-servers 127.0.0.1 )
elif [ "$MODE" = whonix ] && ifconfig virbr1 | grep -q 10.0.2.2 ; then
    LARGS+=( --dns-ipv4-addr 10.0.2.15:9053 --dns-servers 10.0.2.15:9053 )
else
    debug 127.0.0.1:53 not running MODE=$MODE
fi

if ! uname -a | grep -q 'Devuan\|Debian' ; then
    if [ -f $HOME/.local/ ] ; then
	[ -f $HOME/.local/alt.svc ] || touch $HOME/.local/alt.svc
	LARGS+=(  --alt-svc $HOME/.local/alt.svc )
	# #define CURLALTSVC_H2           (1<<4)
    fi
    export CURLOPT_ALTSVC_CTRL=16
fi

declare -a RARGS
RARGS=("$@")
DBUG "$#" "${RARGS[*]}" >&2

i=0
while [ $i -le $RETRIES ] ; do
    # assumes one URL
    if [ "${#RARGS[@]}" -eq 1 ] ; then
	the_url=`sed -e 's@http://@https://@g' -e 's@https*://distfiles.gentoo.org@https://gentoo.osuosl.org@g' -e 's@https*://gentoo.osuosl.org@https://mirror.leaseweb.com/gentoo@g' <<< "${RARGS[*]}"`
    else
	the_url=`sed -e 's@http://@https://@' -e 's@https*://distfiles.gentoo.org@https://gentoo.osuosl.org@g' -e 's@https*://gentoo.osuosl.org@https://mirror.leaseweb.com/gentoo@g' <<< "${RARGS[-1]}"`
    fi
    RARGS[-1]="$the_url"
    site=`sed -e 's@https*://@@g' -e 's@/.*@@' <<< $the_url`

    i=`expr $i + 1`
    if [ "$X" = 1 ] ; then
	rel_file=$( sed -e 's@^file://*@@' -e 's@^https*://*@@' -e 's@[&?#].*@@' <<< $the_url )
	rel_dir=$( sed -e 's@/$@@' <<< $rel_file )
	rel_dir=$( sed -e 's@/[^/]*$@@' <<< $rel_dir )
	[ -d "$P/$rel_dir" ] || mkdir -p "$P/$rel_dir"
	output=`sed -e 's/[!:?#]/_/g' <<< "$P/$rel_file"`
	LARGS+=( --output "$output" --create-dirs )
    fi

    DBUG $EXE "${LARGS[@]}" "${RARGS[@]}" >&2
    echo $EXE "${LARGS[@]}" "${RARGS[@]}" > $LOGF
         $EXE "${LARGS[@]}" "${RARGS[@]}" >> $LOGF 2>&1
    retval=$?

    if [ "$retval" -eq 22 ] || \
	   tail -4 $LOGF | grep -q 'The requested URL returned error:'; then
	# on 22 - change to HTTP code
	code=`tail -4 $LOGF | grep 'The requested URL returned error:' | sed -e 's/.*returned error: //' -e 's/ *$//'`
	if [ "$code" = 416 ] ; then
    	    INFO "$prog retval=$retval code=$code ${HTTP_RESPONSE[416]} $the_url = $LOGF" >&2
	    retval=$code
	elif [ "$code" = 429 ] ; then
    	    ERROR "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
	    retval=$code
	    exit $retval
	elif [ -n "$code" ] && [ "$code" -ge 400 ] ; then
	    # 403 Cloudflare
    	    ERROR "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
	    retval=$code
	elif [ -n "$code" ] && [ "$code" -lt 400 ] ; then
    	    INFO "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
	else
    	    WARN "$prog retval=$retval \"$code\" $the_url = $LOGF" >&2
	fi

    elif [ "$retval" = 35 ] ; then
	# 35 CURLE_SSL_CONNECT_ERROR
	ERROR "$prog retval=$retval CURLE_SSL_CONNECT_ERROR $the_url = $LOGF" >&2
	# feedback to scurl_urls.sh
	NOTLSV3+=( $site )

    elif [ "$retval" = 1 ] ; then
	# retval=1 CURLE=CURLE_UNSUPPORTED_PROTOCOL - seems to be a transient error
	WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2
	continue

    elif [ "$retval" = 92 ] ; then
	# curl: (92) HTTP/2 stream 0 was not closed cleanly: INTERNAL_ERROR (err 2)
	WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2
	continue

    elif [ "$retval" -ne 0 ] ; then
	# curl: (3) URL using bad/illegal format or missing URL - worked
	WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2

    elif tail -3 $LOGF | grep -q "HTTP code 504 from proxy after CONNECT" ; then
	WARN "$prog HTTP code 504 from proxy after CONNECT $the_url = $LOGF" >&2
	continue

    elif tail -3 $LOGF | grep -q "503 - Forwarding failure" ; then
	WARN "$prog 503 - Forwarding failure $the_url = $LOGF" >&2
	continue

    else
	INFO  "$prog $output = $LOGF" >&2
#	rm -f $LOGF
    fi
    break
    #  "$P/$rel_file"
    # if [ $retval -gt 0 ] ; then
    # The requested URL returned error: 416
    # if [ $retval = 22 ] && [ "$code" = 416 ] && [ -f "$P/$rel_file" ] ; then
    # fi
done

exit $retval