#!/bin/bash -e # -*- mode: sh; tab-width: 8; coding: utf-8-unix -*- prog=$( basename $0 .bash ) PREFIX=/usr/local ROLE=base shopt -o -s pipefail DEBUG=1 . /usr/local/bin/usr_local_tput.bash [ $( id -u ) -eq 0 ] || { ERROR "this must be run as root" ; exit 1 ; } . /usr/local/bin/proxy_export.bash WD=$PWD MV=mv COPY="ln -s" PYVER=3 PYTHON_MINOR=$( python$PYVER --version 2>&1| sed -e 's@^.* @@' -e 's@\.[0-9]*$@@' ) [ -z "$BASE_PYTHON2_MINOR" ] && \ BASE_PYTHON2_MINOR=$( python2 --version 2>&1| sed -e 's@^.* @@' -e 's@\.[0-9]*$@@' ) [ -z "$BASE_PYTHON3_MINOR" ] && \ BASE_PYTHON3_MINOR=$( python3 --version 2>&1| sed -e 's@^.* @@' -e 's@\.[0-9]*$@@' ) if [ -z "$LIB" -a -d /usr/lib/python$PYTHON_MINOR/site-packages ] ; then LIB=lib elif [ -z "$LIB" -a -d /usr/lib64/python$PYTHON_MINOR/site-packages ] ; then LIB=lib64 elif [ -z "$LIB" -a -d /usr/lib/python$PYTHON_MINOR/dist-packages ] ; then LIB=lib mkdir -p /usr/local/lib/python$PYTHON_MINOR/site-packages ln -s /usr/local/lib/python$PYTHON_MINOR/dist-packages \ /usr/local/lib/python$PYTHON_MINOR/site-packages elif [ -z "$LIB" -a -d /usr/lib/python$PYTHON_MINOR/dist-packages ] ; then LIB=lib64 mkdir -p /usr/local/lib64/python$PYTHON_MINOR/site-packages ln -s /usr/local/lib64/python$PYTHON_MINOR/dist-packages \ /usr/local/lib64/python$PYTHON_MINOR/site-packages elif [ -z "$LIB" -a -d /usr/lib/python$PYVER/dist-packages ] ; then LIB=lib mkdir -p /usr/local/lib/python$PYTHON_MINOR/site-packages ln -s /usr/local/lib/python$PYTHON_MINOR/dist-packages \ /usr/local/lib/python$PYTHON_MINOR/site-packages elif [ -z "$LIB" -a -d /usr/lib/python$PYVER/dist-packages ] ; then LIB=lib64 mkdir -p /usr/local/lib64/python$PYTHON_MINOR/site-packages ln -s /usr/local/lib64/python$PYTHON_MINOR/dist-packages \ /usr/local/lib64/python$PYTHON_MINOR/site-packages fi if [ -z "$LIB" ] ; then ERROR LIB=$LIB empty - no /usr/lib*/python$PYTHON_MINOR/site-packages exit 3 elif [ -n "$LIB" -a ! -d /usr/$LIB/python$PYTHON_MINOR/site-packages ] ; then ERROR LIB=$LIB but no /usr/$LIB/python$PYTHON_MINOR/site-packages exit 3 fi INFO LIB=$LIB /usr/$LIB/python$PYTHON_MINOR/site-packages [ -z "$UPTMP" ] && UPTMP=$PREFIX/tmp # With packer the files we need are not on the host - they are pushed up and $UPTMP is populated with: PDIRS="authorized_keys archives boxuser_pip_cache root_pip_cache cacert.pem wheels" # With vagrant the files may have been tarred on the host and be in their cannonical positions. # We symlink to files under vagrant to /tmp to leave the packer scripts untouched. # With packer and docker we can remote mount partitions and not even copy them up to the guest. [ -n "$TESTF_DEBIAN10_VAR_APT_ARCHIVES" ] && [ -d "$TESTF_DEBIAN10_VAR_APT_ARCHIVES/" ] && \ [ ! -e $UPTMP/archives ] && ln -s $TESTF_DEBIAN10_VAR_APT_ARCHIVES/ $UPTMP/archives ln -s $TESTF_DEBIAN10_VAR_APT_ARCHIVES/*.deb /var/cache/apt/archives 2>/dev/null [ -n "$HOSTVMS_BOXUSER_PLAY_PIP_CACHE" ] && [ -e "$HOSTVMS_BOXUSER_PLAY_PIP_CACHE" ] && \ [ ! -e $UPTMP/boxuser_pip_cache ] && ln -s $HOSTVMS_BOXUSER_PLAY_PIP_CACHE/ $UPTMP/boxuser_pip_cache [ -n "$HOSTVMS_ROOT_PLAY_PIP_CACHE" ] && [ -d "$HOSTVMS_ROOT_PLAY_PIP_CACHE/" ] && \ [ ! -e $UPTMP/root_pip_cache ] && ln -s "$HOSTVMS_ROOT_PLAY_PIP_CACHE/" $UPTMP/root_pip_cache [ -d /usr/local/etc/testforge ] || mkdir -p /usr/local/etc/testforge export PLAY_PIP_CERT="/usr/local/etc/ssl/cacert-testforge.pem" [ -f $PLAY_PIP_CERT ] && \ [ ! -e $UPTMP/cacert.pem ] && ln -s $PLAY_PIP_CERT $UPTMP/cacert.pem # config_file = os.environ.get('PIP_CONFIG_FILE', None) # /usr/$LIB/python2.7/site-packages/pip/_internal/configuration.py bootstrap_mkdir () { mkdir -p $1 ; chgrp $BOX_ALSO_GROUP $1 ; } [ -d /usr/local/tmp ] || { mkdir -p /usr/local/tmp ; chmod 1777 /usr/local/tmp ; } site_packages=$PREFIX/$LIB/python$PYTHON_MINOR/site-packages [ -d $site_packages ] || bootstrap_mkdir $site_packages [ -f $site_packages/__init__.py ] || touch $site_packages/__init__.py if [ ! -d /usr/local/tmp/wheels ] ; then cd /usr/local sh sbin/bootstrap_wheels.bash || exit 4 fi [ ! -d $UPTMP/wheels/ ] && [ $UPTMP/ != /usr/local/tmp/ ] && \ ln -s /usr/local/tmp/wheels $UPTMP/wheels # But with vagrant or docker we may have mounted the HOST partitions that contain the files # [ -z "$TESTF_UBUNTU16_VAR_APT_ARCHIVES" ] && TESTF_UBUNTU16_VAR_APT_ARCHIVES -> $UPTMP/archives [ -d /etc/portage -a -z "$BOX_USER_NAME" ] && BOX_USER_NAME=vagrant [ -d /etc/apt -a -z "$BOX_USER_NAME" ] && BOX_USER_NAME=devuan [ -z "$BOX_USER_HOME" ] && BOX_USER_HOME=/home/$BOX_USER_NAME [ -z "$BOX_ALSO_GROUP" ] && BOX_ALSO_GROUP=adm [ -z "$LOGDIR" ] && LOGDIR=$PREFIX/tmp [ -d $LOGDIR ] || { mkdir $LOGDIR ; chmod 1777 $LOGDIR ; } # not needed: --no-binary :all: --upgrade-strategy only-if-needed # not yet: --user PIP_ARGS="" PIP_INSTALL_ARGS="--disable-pip-version-check --prefix=$PREFIX" scripts="ansible ansible-playbook ansible-pull ansible-doc ansible-galaxy ansible-console ansible-connection ansible-vault" [ -d /etc/apt ] && export DEBIAN_FRONTEND=noninteractive export PIP_DEFAULT_TIMEOUT=60 ANSIBLE_VER="2.9.10" #2? PYYAML_VER="3.12" ansible_tgz=ansible-$ANSIBLE_VER.tar.gz #2? yaml_tgz=PyYAML-$PYYAML_VER.tar.gz if [ -n "$BOX_USER_NAME" ] ; then # Packer will not have created this and we will need it early. [ -d $BOX_USER_HOME ] || \ bootstrap_mkdir $BOX_USER_HOME #? useradd -d $BOX_USER_HOME -G root -m $BOX_USER_NAME # If you want to use your own private key for packer [ -d $BOX_USER_HOME/.ssh ] || \ bootstrap_mkdir $BOX_USER_HOME/.ssh if [ -f $UPTMP/authorized_keys ] ; then $COPY $UPTMP/authorized_keys $BOX_USER_HOME/.ssh && \ chmod 600 $BOX_USER_HOME/.ssh/authorized_keys fi chmod 700 $BOX_USER_HOME/.ssh/ fi [ -d /etc/apt -a -d /var/cache/apt/archives ] || mkdir -p /var/cache/apt/archives # If you upload your cache of Ubuntu .debs, it cuts down on the downloading [ -d $UPTMP/archives ] && \ $COPY $UPTMP/archives/*.deb /var/cache/apt/archives 2>/dev/null # leave this for cleanup: # rm -rf $UPTMP/archives # If you upload your cache of pip files, it cuts down on the downloading if [ -d $UPTMP/boxuser_pip_cache ] ; then bootstrap_mkdir $BOX_USER_HOME/.cache/ && \ cp -rip $UPTMP/boxuser_pip_cache $BOX_USER_HOME/.cache/pip && \ chown -R ${BOX_USER_NAME}.{BOX_ALSO_GROUP} $BOX_USER_HOME/.cache/pip && \ chmod -R g+rw $BOX_USER_HOME/.cache/pip && \ chmod -R o-w $BOX_USER_HOME/.cache/pip fi if [ -d $UPTMP/root_pip_cache ] ; then bootstrap_mkdir /root/.cache/ && \ cp -rip $UPTMP/root_pip_cache /root/.cache/pip && \ chown -R root.root /root/.cache/pip && \ chmod -R g+rw /root/.cache/pip && \ chmod -R o-w /root/.cache/pip fi if [ -d /etc/apt ] ; then if ! route | grep -q ^default ; then DBUG "Not connected; skipping apt-get update" elif [ ! -f /var/log/dpkg.log ] ; then apt-get update # || exit 4 fi which unzip || ! [ -f /var/cache/apt/archives/unzip_*_amd64.deb ] || \ dpkg -i /var/cache/apt/archives/unzip_*_amd64.deb which curl || [ ! -f /var/cache/apt/archives/curl_*_amd64.deb ] || \ dpkg -i /var/cache/apt/archives/curl_*_amd64.deb \ /var/cache/apt/archives/libcurl4_*_amd64.deb \ /var/cache/apt/archives/libcurl4-openssl-dev_*_amd64.deb apt-get install -y --force-yes wget unzip openssl || true [ -f /usr/include/Python.h ] || \ apt-get install -y --force-yes \ libffi-dev libssl-dev python3-dev python3-pycparser \ python3-coverage || \ echo WARN you must run apt-get update # msg: Could not find `coverage` module. ?python3-apt ? elif [ -d /etc/portage ] ; then # FixMe: put these in wheels? [ -x /usr/bin/unzip ] || which unzip 2>/dev/null || emerge -vb app-arch/unzip [ -x /usr/bin/wget ] || which wget 2>/dev/null || emerge -vb net-misc/wget which openssl 2>/dev/null || timeout 600 emerge -vb dev-libs/openssl # openssl installs: # dev-python/pyopenssl-19.1.0 # dev-python/six-1.13.0 # dev-python/cryptography-2.8 # dev-python/cffi-1.12.3:0/1.12.3 # dev-python/pycparser-2.19-r1 # dev-python/ply-3.11:0/3.11 # virtual/python-ipaddress-1.0-r1 # dev-python/ipaddress-1.0.23 # virtual/python-enum34-2 # dev-python/enum34-1.1.6-r1 python$PYVER -c 'import OpenSSL' 2>/dev/null || \ timeout 600 emerge -vb dev-python/pyopenssl python$PYVER -c 'import pycparser' 2>/dev/null || \ timeout 600 emerge -vb dev-python/pycparser python$PYVER -c 'import yaml' 2>/dev/null || \ timeout 600 emerge -vb dev-python/pyyaml DBUG "Gentoo Installed openssl and wget" fi # On a CORP laptop off the VPN we may need some CAs [ -d $PREFIX/etc/ssl ] || mkdir -p $PREFIX/etc/ssl [ ! -f $PLAY_PIP_CERT ] && \ [ -f $UPTMP/cacert.pem ] && \ $COPY $UPTMP/cacert.pem $PLAY_PIP_CERT # pip gets confused # or just delete $PREFIX/$LIB/python$PYTHON_MINOR/dist-packages afterwards PYTHON_MINOR=$( python$PYVER --version 2>&1| sed -e 's@^.* @@' -e 's@\.[0-9]*$@@' ) site_packages=$PREFIX/$LIB/python$PYTHON_MINOR/site-packages [ -d $site_packages ] || bootstrap_mkdir $site_packages [ -f $site_packages/__init__.py ] || touch $site_packages/__init__.py if [ -d /etc/apt ] ; then dist_packages=$PREFIX/lib/python$PYTHON_MINOR/dist-packages WD=$PWD if [ -d $dist_packages ] ; then cd $PREFIX/lib/python$PYTHON_MINOR ln -s $site_packages . cd $WD fi fi # we will use $PREFIX/bin/python3.bash NOT $PREFIX/bin/python3.sh # to not conflict with what Ansible will push later/before. if [ ! -e $PREFIX/bin/python$PYVER.bash ] ; then INFO "bootstrapping $PREFIX/bin/python$PYVER.bash" cat > $PREFIX/bin/python$PYVER.bash << EOF #!/bin/sh # -*-mode: sh; tab-width: 8; coding: utf-8-unix -*- # from bootstrap_pip_ansible.bash . /usr/local/bin/usr_local_tput.bash || exit 2 PREFIX=/usr/local # pip gets confused dist_packages=$site_packages dist_packages=\$dist_packages:\${dist_packages}/pip/_vendor if [ -z "$PYTHONPATH" ] ; then export PYTHONPATH=\$dist_packages else export PYTHONPATH=\$PYTHONPATH:\$dist_packages fi exec python$PYVER "\$@" EOF chmod 755 $PREFIX/bin/python$PYVER.bash fi # pip may be loaded in the base iso if [ -x $PREFIX/bin/python$PYVER.bash ] && \ $PREFIX/bin/python$PYVER.bash -c 'import pip' 2>/dev/null ; then INFO pip$VER already installed elif [ ! -d $UPTMP/wheels/ ] ; then WARN $UPTMP/wheels not found else # we may be without the VPN/proxy but on a corporate laptop # with a hosed chain of Certificate Authorities for the MITM proxy # in which case http://bootstrap.pypa.io/get-pip.py will not work, # so effective but groddy: # just unzip the wheels into site-packages and force-reinstall later cd $UPTMP/wheels/ INFO "installing pip - unzipping wheels into $site_packages" for file in *.whl ; do #a=$( echo $file | sed -e 's/-.*//' ) #b=$( basename $a|sed -e 's/Py//'|tr '[A-Z]' '[a-z]' ) #python$PYVER -c "import $b" 2>/dev/null >/dev/null && continue unzip -n $file -d $site_packages >/dev/null done # morons # -rwx------ 1 root root 8866 Jun 11 2018 /usr/local/$LIB/python$PYTHON_MINOR/site-packages/idna-2.7.dist-info/METADATA find $site_packages -type d -exec chmod a+rx '{}' \; find $site_packages -type f -exec chmod a+r '{}' \; chgrp -R "$BOX_ALSO_GROUP" $site_packages # hack in a PYTHONPATH for our unzipped wheels - removed later for elt in pip ; do # is wheel needed? INFO "Installing $elt" # use $PYVER.bash for bootstrap - $PYVER.bash will come later [ -f $PREFIX/bin/$elt$PYVER.bash ] || \ cat > $PREFIX/bin/$elt$PYVER.bash << EOF #!/bin/sh # -*-mode: sh; tab-width: 8; coding: utf-8-unix -*- export PLAY_PIP_CERT=$PIP_CERT export PYTHONPATH=${site_packages} export PYTHONPATH=\$PYTHONPATH:${site_packages}/pip/_vendor #? FixMe: narrow to InsecurePlatformWarning python$PYVER -W ignore -m $elt "\$@" EOF chmod 755 $PREFIX/bin/$elt$PYVER.bash $PREFIX/bin/$elt$PYVER.bash --help >/dev/null DBUG "Installed $elt$PYVER.bash" done fi # do I still need this #if [ -x $PREFIX/bin/pip$PYVER ] && [ -d $site_packages ] ; then # export PYTHONPATH=$site_packages:$site_packages/pip/_vendor #fi if [ ! -x $PREFIX/bin/pip$PYVER.sh ] ; then ERROR "Failed to Install pip$PYVER at $PREFIX/bin/pip$PYVER.sh" exit 3 elif ! $PREFIX/bin/python$PYVER.bash -m pip -V ; then ERROR "Failed to run pip$PYVER at $PREFIX/bin/pip$PYVER" exit 4 fi if [ -f $PLAY_PIP_CERT ] ; then if [ ! -f $site_packages/pip/_vendor/requests/cacert.pem.dst ] && \ [ -f $site_packages/pip/_vendor/requests/cacert.pem ] && \ [ ! -h $site_packages/pip/_vendor/requests/cacert.pem ] ; then mv $site_packages/pip/_vendor/requests/cacert.pem \ $site_packages/pip/_vendor/requests/cacert.pem.dst fi if [ ! -h $site_packages/pip/_vendor/requests/cacert.pem ] ; then rm -f $site_packages/pip/_vendor/requests/cacert.pem fi [ -e $site_packages/pip/_vendor/requests/cacert.pem ] || \ ln -s $PLAY_PIP_CERT $site_packages/pip/_vendor/requests/cacert.pem INFO linked $PLAY_PIP_CERT $site_packages/pip/_vendor/requests/cacert.pem fi # dont use -CAfile $UPTMP/cacert.pem - we want it to fail if we need the cert if ! route | grep -q ^default ; then DBUG "Not connected; skipping SSL Certificate Authority chain" elif [ -n "$https_proxy" ] ; then proxy=`echo "$https_proxy" | sed -e 's/https*:\/*//'` openssl s_client -connect pypi.org:443 --proxy $proxy > $LOGDIR/pip_install_pip_ansible.log 2>&1 || { ERROR pip$PYVER.sh $PIP_ARGS install $PIP_INSTALL_ARGS $ansible_tgz tail $LOGDIR/pip_install_pip_ansible.log exit 8 } return 0 } boostrap_patch_ansible () { local WD=$PWD [ -d /usr/local/patches/base ] || return 0 [ -f /usr/local/sbin/base_patch_from_diff.bash ] || return 0 cd /usr/local/patches/base || return 1 [ -d usr/local/src/ansible-$ANSIBLE_VER ] || return 0 # this vacuumns all diff files below the root /usr/local/sbin/base_patch_from_diff.bash usr/local/src/ansible-$ANSIBLE_VER return 0 } boostrap_setup_ansible () { local WD=$PWD cd /usr/local/src [ -d ansible-$ANSIBLE_VER ] || tar xfz $UPTMP/wheels/$ansible_tgz cd ansible-$ANSIBLE_VER /usr/local/sbin/base_patch_from_diff.bash usr/local/src/ansible-$ANSIBLE_VER RARGS=" --user $RARGS" # RARGS=" --install-layout=unix $RARGS" export PYTHONPATH=/usr/local/$LIB/python$PYTHON_MINOR/site-packages DBUG "/usr/local/bin/python$PYVER.bash setup.py install $RARGS" sudo -u $"$BOX_USER_NAME" \ /usr/local/bin/python$PYVER.bash setup.py install $RARGS \ >> install.log retval=$? cd $WD return $retval } # NOW we use our fresh pip to install ansible from source, into /usr/local if [ -d $PREFIX/src/ansible-$ANSIBLE_VER ] ; then INFO already installed $PREFIX/src/ansible-$ANSIBLE_VER elif [ ! -f $UPTMP/wheels/$ansible_tgz ] ; then ERROR tgz missing $UPTMP/wheels/$ansible_tgz exit 7 else if false ; then boostrap_pip_ansible else boostrap_setup_ansible [ $? -eq 0 ] || { ERROR installing ansible ; tail install.log ; exit 8 ; } fi boostrap_patch_ansible if [ -d /etc/portage/ ] ; then [ -d /etc/portage/profile ] || mkdir /etc/portage/profile grep -q app-admin/ansible-$ANSIBLE_VER /etc/portage/profile/package.provided || \ echo app-admin/ansible-$ANSIBLE_VER >> /etc/portage/profile/package.provided fi cd $PREFIX/bin [ -e ansible-doc ] || { ERROR installing ansible-doc ; exit 9 ; } grep "#\!.$PREFIX/bin/python$PYVER.bash" ansible-doc || \ sed -e "s@^#\!.*python.*@#\!${PREFIX}/bin/python$PYVER.bash@" -i $scripts fi ansible --version || exit 10 if [ -f $PLAY_PIP_CERT ] ; then export PLAY_PIP_CERT=$PIP_CERT PIP_INSTALL_ARGS="$PIP_INSTALL_ARGS --cert $PLAY_PIP_CERT" else WARN "PLAY_PIP_CERT not found $PIP_CERT" fi [ ! -f /etc/wgetrc ] || sh $WD/bootstrap_proxy.bash # pip uses curl - and has a config file PIP_CONFIG DBUG "http_proxy=$http_proxy https_proxy=$https_proxy" if [ -n "$https_proxy" ] ; then INFO "Adding to PIP_INSTALL_ARGS --proxy=$https_proxy" elif [ -f /etc/wgetrc ] && grep ^http_proxy /etc/wgetrc ; then proxy=$( grep ^http_proxy /etc/wgetrc|sed -e 's@.*=@--proxy=@' ) INFO "Adding to PIP_INSTALL_ARGS $proxy" PIP_INSTALL_ARGS="$PIP_INSTALL_ARGS $proxy" fi cd $PREFIX/src # install pycurl as a test of pip and a requisite for proxyauth.py if ! $PREFIX/bin/python$PYVER.bash -c 'import curl' 2>/dev/null ; then if [ -d /etc/apt ] ; then apt-get install -y --force-yes libcurl4-openssl-dev \ 2>&1 | tee $LOGDIR/apt-get_install_libcurl4-openssl-dev.log elif [ -d /etc/portage ] ; then [ -x /usr/bin/curl ] || which curl 2>/dev/null || emerge -vb curl fi #? --allow-unverified pycurl if ! route | grep -q ^default ; then INFO "Not connected; not installing pycurl" elif $PREFIX/bin/pip$PYVER.sh install $PIP_INSTALL_ARGS pycurl >> $LOGDIR/pip_install_pycurl.log 2>&1 ; then INFO "Installed pycurl from pip with $PREFIX/bin/pip install $PIP_INSTALL_ARGS" # We dont fail the packer build if it errors - just fix it and rerun $PREFIX/bin/python$PYVER.bash -c 'import curl; print curl.__file__' || true else WARN "Installing pycurl failed with $PREFIX/bin/pip install $PIP_INSTALL_ARGS" cat $LOGDIR/pip_install_pycurl.log fi fi [ -e /usr/local/bin/python$PYVER.sh ] || \ [ -h /usr/local/bin/python$PYVER.sh ] || \ ln -s /usr/local/bin/python$PYVER.bash /usr/local/bin/python$PYVER.sh find /usr/local/$LIB/python$PYTHON_MINOR/site-packages/ansible/modules/ -name \*.py \ -exec grep -q /usr/bin/python '{}' \; -print \ -exec sed -e "1,3s@#!/usr/bin/python@#!/usr/local/bin/python$PYVER.bash@" -i '{}' \; exit 0