emdee/proxy_role
1
0
Fork 0
Code Issues Pull Requests Wiki Activity
d29b1e4542
proxy_role/overlay/Linux/usr/local/bin/scurl.bash
emdee 757ca483f0 first
2024-01-06 01:57:28 +00:00

255 lines
7.5 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# -*- mode: sh; tab-width: 8; coding: utf-8-unix -*-
# must not use stdout
prog=$( basename $0 .bash )
PREFIX=/usr/local
ROLE=proxy
prog=scurl
umask 022
RETRIES=2
SSL_VER=3
. /usr/local/bin/proxy_ping_lib.bash
[ -f /usr/local/bin/proxy_curl_lib.bash ] && \
. /usr/local/bin/proxy_curl_lib.bash
if ! route | grep -q ^def ; then
WARN $prog we are not connected >&2
exit -1
fi
usage="curls with some wget options
Usage: $prog options -- curl-options
Options:
-P, --directory-prefix
-X, --force-directories create directories to download to
-C, --cacert CA certs in .pem
-M, --mode proxy_ping_mode
-S, --ssl ssl version 2=tls1.2 3=tls1.3
-Y, --ciphers comma sep list of ciphers
-Q, --quiet --silent --show-error
-h, --help display this help and exit
"
if [[ $? -ne 0 ]]; then
echo "$usage"
exit 2
fi
declare -a LARGS
# --location is required to follow redirects
# im not sure about --http2
LARGS+=( --remote-time --location --max-redirs 10 --continue-at - )
LARGS+=( --retry-delay 10 --show-error --fail )
# --proto-redir https --proto =https is required to prevent protocol downgrades
LARGS+=( --proto-redir https --proto-default https --proto =https )
[ -z "$MODE" ] && MODE=$( /usr/local/bin/proxy_ping_lib.bash proxy_ping_mode )
[ -z "$socks_proxy" ] && . /usr/local/bin/proxy_export.bash
SSL_LIB=openssl # nss
if [ -x /var/local/bin/curl.bash ] ; then
EXE=/var/local/bin/curl.bash
elif which scurl ; then
EXE=`which scurl`
else
EXE=curl
fi
SHORT=M:QP:XC:F:hS:
LONG=mode:,quiet,directory-prefix:,force-directories,cacert,ca-cert:,help,ssl:
#? export POSIXLY_CORRECT=1
PARSED=$(getopt --options $SHORT --longoptions $LONG --name "$prog" -- "$@")
eval set -- "$PARSED"
P="$PWD"
X="0"
# echo DEBUG: WD=$WD rest=$*
LOGF=/tmp/$prog$$.err
SSL_CIPHERS=""
while true; do
case "$1" in
-P|--directory-prefix)
shift
P="$1"
shift
# echo DEBUG: P=$WD rest=$*
;;
-X|--force-directories)
X=1
shift
;;
-S|--ssl)
shift
SSL_VER="$1"
shift
;;
-Y|--ciphers)
shift
SSL_CIPHERS="$1"
shift
;;
-C|--cacert|-Z|--ca-cert)
shift
CA_CERT="$1"
shift
;;
-M|--mode)
shift
MODE="$1"
shift
;;
-Q|--quiet)
shift
LARGS="$LARGS --silent --show-error"
;;
-h|--help)
echo USAGE: "$usage"
exit 0
;;
--)
shift
break
;;
*)
# echo ERROR: unhandled arguments $* - use -- after -P $PWD or -X ; exit 3
break
;;
esac
done
[ "$SSL_VER" -ge 2 -a "$SSL_VER" -le 3 ] || { ERROR "SSL_VER $SSL_VER" ; exit 6 ; }
LARGS+=( --tlsv1.$SSL_VER )
if [ -n "$SSL_CIPHERS" -a "$SSL_VER" = 2 ] ; then
[ $SSL_LIB = openssl ] && \
SSL_CIPHERS="ECDHE-RSA-AES256-SHA" # ECDHE-RSA-AES256-GCM-SHA384
[ $SSL_LIB = nss ] && \
SSL_CIPHERS="ecdhe_rsa_aes_256_sha"
fi
if [ -n "$SSL_CIPHERS" -a "$SSL_VER" = 3 ] ; then
[ $SSL_LIB = openssl ] && \
SSL_CIPHERS="TLS_AES_256_GCM_SHA384" # TLS_CHACHA20_POLY1305_SHA256
[ $SSL_LIB = nss ] && \
SSL_CIPHERS="aes_256_gcm_sha_384"
fi
[ -n "$SSL_CIPHERS" ] && LARGS+=( --ciphers "$SSL_CIPHERS" )
if [ "$MODE" = tor -o "$MODE" = selektor -o "$MODE" = whonix ] && \
netstat -nle4 | grep -q 127.0.0.1:53 ; then
LARGS+=( --dns-ipv4-addr 127.0.0.1 --dns-servers 127.0.0.1 )
elif [ "$MODE" = whonix ] && ifconfig virbr1 | grep -q 10.0.2.2 ; then
LARGS+=( --dns-ipv4-addr 10.0.2.15:9053 --dns-servers 10.0.2.15:9053 )
else
debug 127.0.0.1:53 not running MODE=$MODE
fi
if ! uname -a | grep -q 'Devuan\|Debian' ; then
if [ -f $HOME/.local/ ] ; then
[ -f $HOME/.local/alt.svc ] || touch $HOME/.local/alt.svc
LARGS+=( --alt-svc $HOME/.local/alt.svc )
# #define CURLALTSVC_H2 (1<<4)
fi
export CURLOPT_ALTSVC_CTRL=16
fi
declare -a RARGS
RARGS=("$@")
DBUG "$#" "${RARGS[*]}" >&2
i=0
while [ $i -le $RETRIES ] ; do
# assumes one URL
if [ "${#RARGS[@]}" -eq 1 ] ; then
the_url=`sed -e 's@http://@https://@g' -e 's@https*://distfiles.gentoo.org@https://gentoo.osuosl.org@g' -e 's@https*://gentoo.osuosl.org@https://mirror.leaseweb.com/gentoo@g' <<< "${RARGS[*]}"`
else
the_url=`sed -e 's@http://@https://@' -e 's@https*://distfiles.gentoo.org@https://gentoo.osuosl.org@g' -e 's@https*://gentoo.osuosl.org@https://mirror.leaseweb.com/gentoo@g' <<< "${RARGS[-1]}"`
fi
RARGS[-1]="$the_url"
site=`sed -e 's@https*://@@g' -e 's@/.*@@' <<< $the_url`
i=`expr $i + 1`
if [ "$X" = 1 ] ; then
rel_file=$( sed -e 's@^file://*@@' -e 's@^https*://*@@' -e 's@[&?#].*@@' <<< $the_url )
rel_dir=$( sed -e 's@/$@@' <<< $rel_file )
rel_dir=$( sed -e 's@/[^/]*$@@' <<< $rel_dir )
[ -d "$P/$rel_dir" ] || mkdir -p "$P/$rel_dir"
output=`sed -e 's/[!:?#]/_/g' <<< "$P/$rel_file"`
LARGS+=( --output "$output" --create-dirs )
fi
DBUG $EXE "${LARGS[@]}" "${RARGS[@]}" >&2
echo $EXE "${LARGS[@]}" "${RARGS[@]}" > $LOGF
$EXE "${LARGS[@]}" "${RARGS[@]}" >> $LOGF 2>&1
retval=$?
if [ "$retval" -eq 22 ] || \
tail -4 $LOGF | grep -q 'The requested URL returned error:'; then
# on 22 - change to HTTP code
code=`tail -4 $LOGF | grep 'The requested URL returned error:' | sed -e 's/.*returned error: //' -e 's/ *$//'`
if [ "$code" = 416 ] ; then
INFO "$prog retval=$retval code=$code ${HTTP_RESPONSE[416]} $the_url = $LOGF" >&2
retval=$code
elif [ "$code" = 429 ] ; then
ERROR "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
retval=$code
exit $retval
elif [ -n "$code" ] && [ "$code" -ge 400 ] ; then
# 403 Cloudflare
ERROR "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
retval=$code
elif [ -n "$code" ] && [ "$code" -lt 400 ] ; then
INFO "$prog retval=$retval code=$code ${HTTP_RESPONSE[$code]} $the_url = $LOGF" >&2
else
WARN "$prog retval=$retval \"$code\" $the_url = $LOGF" >&2
fi
elif [ "$retval" = 35 ] ; then
# 35 CURLE_SSL_CONNECT_ERROR
ERROR "$prog retval=$retval CURLE_SSL_CONNECT_ERROR $the_url = $LOGF" >&2
# feedback to scurl_urls.sh
NOTLSV3+=( $site )
elif [ "$retval" = 1 ] ; then
# retval=1 CURLE=CURLE_UNSUPPORTED_PROTOCOL - seems to be a transient error
WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2
continue
elif [ "$retval" = 92 ] ; then
# curl: (92) HTTP/2 stream 0 was not closed cleanly: INTERNAL_ERROR (err 2)
WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2
continue
elif [ "$retval" -ne 0 ] ; then
# curl: (3) URL using bad/illegal format or missing URL - worked
WARN "$prog retval=$retval CURLE=${CURLE[$retval]} $the_url = $LOGF" >&2
elif tail -3 $LOGF | grep -q "HTTP code 504 from proxy after CONNECT" ; then
WARN "$prog HTTP code 504 from proxy after CONNECT $the_url = $LOGF" >&2
continue
elif tail -3 $LOGF | grep -q "503 - Forwarding failure" ; then
WARN "$prog 503 - Forwarding failure $the_url = $LOGF" >&2
continue
else
INFO "$prog $output = $LOGF" >&2
# rm -f $LOGF
fi
break
# "$P/$rel_file"
# if [ $retval -gt 0 ] ; then
# The requested URL returned error: 416
# if [ $retval = 22 ] && [ "$code" = 416 ] && [ -f "$P/$rel_file" ] ; then
# fi
done
exit $retval
Reference in New Issue View Git Blame Copy Permalink