emdee/libvirt_cloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

added

Browse source
This commit is contained in:
emdee 2024-01-01 01:04:40 +00:00
parent eaf6ffdbef
commit c417a6f3f9
29 changed files with 16625 additions and 608 deletions
Download patch file Download diff file Expand all files Collapse all files

75
src/ansible_gentooimgr/bin/ansible_gentooimgr.bash Executable file
View file

@ -0,0 +1,75 @@
#!/bin/bash
# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
prog=`basename $0 .bash`
PREFIX=/usr/local
[ -f /usr/local/bin/usr_local_tput.bash ] && \
. /usr/local/bin/usr_local_tput.bash || {
DBUG() { echo DEBUG $* ; }
INFO() { echo INFO $* ; }
WARN() { echo WARN $* ; }
ERROR() { echo ERROR $* ; }
}
ROLE=toxcore
BOX=gentoo1
EXE=python3.sh
export PLAY_ANSIBLE_SRC=/o/var/local/src/play_tox
yamllint -c $PLAY_ANSIBLE_SRC/.yamllint.rc $PLAY_ANSIBLE_SRC/hosts.yml || {
ERROR
exit 1
}
[ "$#" -eq 0 ] && set -- build
# put these values in $BASE_SRC_ANSIBLE/hosts.yml
[ -n "$BOX_NBD_FILES" ] || \
BOX_NBD_FILES=$( /usr/local/bin/ansible_get_inventory.bash BOX_NBD_FILES $BOX)
[ -n "$BOX_NBD_BASE_QCOW" ] || \
BOX_NBD_BASE_QCOW=$( /usr/local/bin/ansible_get_inventory.bash BOX_NBD_BASE_QCOW $BOX)
[ -n "$BOX_NBD_LOGLEVEL" ] || \
BOX_NBD_LOGLEVEL=$( /usr/local/bin/ansible_get_inventory.bash BOX_NBD_LOGLEVEL $BOX)
[ -z "$BOX_NBD_LOGLEVEL" ] && BOX_NBD_LOGLEVEL=20
DBUG "BOX_NBD_LOGLEVEL=$BOX_NBD_LOGLEVEL"
declare -a RARGS
RARGS=(-j 1)
if [ -n "$BOX_NBD_FILES" ] && [ -d "$BOX_NBD_FILES" ] ; then
PORTAGE="$BOX_NBD_FILES"/portage-20231223.tar.xz
[ -f $PORTAGE ] && RARGS+=(--portage $PORTAGE)
STAGE3="$BOX_NBD_FILES"/stage3-amd64-openrc-20231217T170203Z.tar.xz
[ -f $STAGE3 ] && RARGS+=(--stage3 $STAGE3)
RARGS+=( --download-dir $BOX_NBD_FILES )
fi
[ -n "$BOX_NBD_BASE_QCOW" ] && \
BOX_NBD_BASE_DIR=$(dirname "$BOX_NBD_BASE_QCOW") && \
[ -d "$BOX_NBD_BASE_DIR" ] && \
RARGS+=( --temporary-dir "$BOX_NBD_BASE_DIR" )
export BOX_NBD_OVERLAY_DIR
LOGLEVEL=20
JSON=$PLAY_ANSIBLE_SRC/src/ansible_gentooimgr/gentooimgr/configs/base.json
if [ -n "$JSON" ] && [ -f "$JSON" ] ; then
PKGS=`yq .packages.base < $JSON`
DBUG gentooimgr "${PKGS}"
RARGS+=( --config "$JSON" )
fi
# default --kernel-dir /usr/src/linux
RARGS+=( --config-base --loglevel $LOGLEVEL --days 27 --openrc --format qcow2 )
DBUG gentooimgr "${RARGS[@]}" "$@"
PYTHONPATH=$PLAY_ANSIBLE_SRC/src/ansible_gentooimgr $EXE \
-m gentooimgr \
"${RARGS[@]}" "$@"
retval=$?
[ $retval -gt 0 ] && exit 1$retval
#[ -f $BOX_NBD_OVERLAY_DIR/images/$BOX_NBD_OVERLAY_NAME.img ] && \
# INFO $BOX_NBD_OVERLAY_DIR/images/$BOX_NBD_OVERLAY_NAME.img || {
# ERROR NO $BOX_NBD_OVERLAY_DIR/images/$BOX_NBD_OVERLAY_NAME.img ; exit 2$retval ; }
exit 0

360
src/ansible_gentooimgr/gentooimgr/#install.py#
View file

@ -1,360 +0,0 @@
"""Configure a Gentoo guest with cloud image settings
This step keeps track of how far it's gotten, so re-running this command
will continue on if an error was to occur, unless --start-over flag is given.
"""
import os
import sys
import shutil
import configparser
from subprocess import Popen, PIPE
import logging
import traceback
import gentooimgr.config
import gentooimgr.configs
import gentooimgr.common
import gentooimgr.chroot
import gentooimgr.kernel
from gentooimgr import LOG
from gentooimgr import HERE
from gentooimgr.configs import *
FILES_DIR = os.path.join(HERE, "..")
def step1_diskprep(args, cfg):
LOG.info("\t:: Step 1: Disk Partitioning")
# http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
# http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
cmds = [
['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
['partprobe'],
['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
]
for c in cmds:
proc = Popen(c, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
completestep(1, "diskprep")
def step2_mount(args, cfg):
LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
proc.communicate()
completestep(2, "mount")
def step3_stage3(args, cfg):
LOG.info(f'\t:: Step 3: Stage3 Tarball')
stage3 = cfg.get("stage3") or args.stage3 # FIXME: auto detect stage3 images in mountpoint and add here
if not stage3:
stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
f'{cfg.get("mountpoint")}'])
proc.communicate()
completestep(3, "stage3")
def step4_binds(args, cfg):
LOG.info(f'\t:: Step 4: Binding Filesystems')
gentooimgr.chroot.bind(verbose=False)
completestep(4, "binds")
def step5_portage(args, cfg):
LOG.info(f'\t:: Step 5: Portage')
portage = cfg.get("portage") or args.portage
if not portage:
portage = gentooimgr.common.portage_from_dir(FILES_DIR)
proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
proc.communicate()
# Edit portage
portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
os.makedirs(portage_env, exist_ok=True)
with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
f.write('MAKEOPTS="-j1"\n')
env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
with open(env_path, 'w') as f:
f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
completestep(5, "portage")
def step6_licenses(args, cfg):
LOG.info(f'\t:: Step 6: Licenses')
license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
os.makedirs(license_path, exist_ok=True)
for f, licenses in cfg.get("licensefiles", {}).items():
with open(os.path.join(license_path, f), 'w') as f:
f.write('\n'.join(licenses))
completestep(6, "license")
def step7_repos(args, cfg):
LOG.info(f'\t:: Step 7: Repo Configuration')
repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
os.makedirs(repo_path, exist_ok=True)
# Copy from template
repo_file = os.path.join(repo_path, 'gentoo.conf')
shutil.copyfile(
os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
repo_file)
# Regex replace lines
cp = configparser.ConfigParser()
for repofile, data in cfg.get("repos", {}).items():
cp.read(cfg.get("mountpoint") + repofile) # repofile should be absolute path, do not use os.path.join.
for section, d in data.items():
if section in cp:
for key, val in d.items():
# Replace everything after the key with contents of value.
# Sed is simpler than using regex for this purpose.
cp.set(section, key, val)
else:
sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
completestep(7, "repos")
def step8_resolv(args, cfg):
LOG.info(f'\t:: Step 8: Resolv')
proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
proc.communicate()
# Copy all step files and python module to new chroot
os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
completestep(8, "resolv")
def step9_sync(args, cfg):
LOG.info(f"\t:: Step 9: sync")
LOG.info("\t\t:: Entering chroot")
os.chroot(cfg.get("mountpoint"))
os.chdir(os.sep)
os.system("source /etc/profile")
proc = Popen(["emerge", "--sync", "--quiet"])
proc.communicate()
LOG.info("\t\t:: Emerging base")
proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
proc.communicate()
completestep(9, "sync")
def step10_emerge_pkgs(args, cfg):
LOG.info(f"\t:: Step 10: emerge pkgs")
packages = cfg.get("packages", {})
for oneshot_up in packages.get("oneshots", []):
proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
proc.communicate()
for single in packages.get("singles", []):
proc = Popen(["emerge", "-j1", single])
proc.communicate()
LOG.info(f"KERNEL PACKAGES {packages.get('kernel')}")
if packages.get("kernel", []):
cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
proc = Popen(cmd)
proc.communicate()
cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
cmd += packages.get("keepgoing", [])
proc = Popen(cmd)
proc.communicate()
cmd = ["emerge", "-j", str(args.threads)]
cmd += packages.get("base", [])
cmd += packages.get("additional", [])
cmd += packages.get("bootloader", [])
LOG.info(cmd)
proc = Popen(cmd)
proc.communicate()
completestep(10, "pkgs")
def step11_kernel(args, cfg):
# at this point, genkernel will be installed
LOG.info(f"\t:: Step 11: kernel")
proc = Popen(["eselect", "kernel", "set", "1"])
proc.communicate()
if not args.kernel_dist:
os.chdir(args.kernel_dir)
threads = str(gentooimgr.config.THREADS)
gentooimgr.kernel.build_kernel(args, cfg)
completestep(11, "kernel")
def step12_grub(args, cfg):
LOG.info(f"\t:: Step 12: kernel")
proc = Popen(["grub-install", cfg.get('disk')])
proc.communicate()
code = proc.returncode
if code != 0:
sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
sys.exit(code)
with open("/etc/default/grub", 'w') as f:
f.write(f"{gentooimgr.kernel.GRUB_CFG}")
proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
proc.communicate()
completestep(12, "grub")
def step13_serial(args, cfg):
LOG.info(f"\t:: Step 13: Serial")
os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
completestep(13, "serial")
def step14_services(args, cfg):
LOG.info(f"\t:: Step 14: Services")
for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
"cloud-final", "ntpd", "nfsclient"]:
if args.profile == "systemd":
proc = Popen(["systemctl", "enable", service])
else:
proc = Popen(["rc-update", "add", service, "default"])
proc.communicate()
completestep(14, "services")
def step15_ethnaming(args, cfg):
LOG.info(f"\t:: Step 15: Eth Naming")
completestep(15, "networking")
def step16_sysconfig(args, cfg):
LOG.info(f"\t:: Step 16: Sysconfig")
with open("/etc/timezone", "w") as f:
f.write("UTC")
proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
proc.communicate()
with open("/etc/locale.gen", "a") as f:
f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
proc = Popen(["locale-gen"])
proc.communicate()
proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
proc.communicate()
proc = Popen(["env-update"])
proc.communicate()
with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
f.write("vm.swappiness = 0\n")
modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
os.makedirs(modloadpath, exist_ok=True)
with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
if not os.path.exists(cloudcfg):
os.makedirs(cloudcfg, exist_ok=True)
os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
cfg.write(f"{CLOUD_YAML}")
os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
tmpl.write(f"{HOST_TMPL}") # FIXME:
os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
proc.communicate()
hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
with open(hostname, 'w') as f:
f.write(f"{HOSTNAME}\n")
os.chmod(hostname, 0o644)
proc = Popen(["eix-update"])
proc.communicate()
os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
completestep(16, "sysconfig")
def step17_fstab(args, cfg):
LOG.info(f"\t:: Step 17: fstab")
with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
completestep(17, "fstab")
def completestep(step, stepname, prefix='/tmp'):
with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
f.write("done.") # text in this file is not currently used.
def getlaststep(prefix='/tmp'):
i = 1
found = False
while not found:
if os.path.exists(f"{i}.step"):
i += 1
else:
found = True
return i
def stepdone(step, prefix='/tmp'):
return os.path.exists(os.path.join(prefix, f"{step}.step"))
def configure(args, config: dict):
# Load configuration
if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
if not args.force:
# We aren't in a gentoo live cd are we?
sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
"If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
sys.exit(1)
else:
# Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
os.makedirs(gentooimgr.config.GENTOO_MOUNT)
# disk prep
cfg = config
# must be root for linux-6.1.52-pentoo/certs/signing_key.pem
if not stepdone(1): step1_diskprep(args, cfg)
# mount root
if not stepdone(2): step2_mount(args, cfg)
# extract stage
if not stepdone(3): step3_stage3(args, cfg)
# mount binds
if not stepdone(4): step4_binds(args, cfg)
# extract portage
if not stepdone(5): step5_portage(args, cfg)
# Set licenses
if not stepdone(6): step6_licenses(args, cfg)
# repos.conf
if not stepdone(7): step7_repos(args, cfg)
# portage env files and resolv.conf
if not stepdone(8): step8_resolv(args, cfg)
# emerge --sync
if not stepdone(9): step9_sync(args, cfg)
# bindist
if not stepdone(10): step10_emerge_pkgs(args, cfg)
# emerge packages
# configure & emerge kernel (use cloud configuration too)
if not stepdone(11): step11_kernel(args, cfg)
# grub
if not stepdone(12): step12_grub(args, cfg)
# enable serial console
if not stepdone(13): step13_serial(args, cfg)
# services
if not stepdone(14): step14_services(args, cfg)
# eth0 naming
# timezone
if not stepdone(15): step15_ethnaming(args, cfg)
# locale
# set some sysctl things
# set some dhcp things
# hostname
if not stepdone(16): step16_sysconfig(args, cfg)
# fstab
if not stepdone(17): step17_fstab(args, cfg)
# copy cloud cfg?
gentooimgr.chroot.unbind()
# Finish install processes like emaint and eix-update and news read

13
src/ansible_gentooimgr/gentooimgr/__main__.py
View file

@ -72,7 +72,7 @@ def main(args):
gentooimgr.kernel.build_kernel(args, configjson)
return 0
if __name__ == "__main__":
def oMainParser()
"""Gentoo Cloud Image Builder Utility"""
parser = argparse.ArgumentParser(prog="gentooimgr", description="Gentoo Image Builder Utility")
parser.add_argument("-c", "--config", nargs='?', type=pathlib.Path,
@ -161,9 +161,9 @@ if __name__ == "__main__":
parser_kernel = subparsers.add_parser('kernel', help="Build the kernel based on configuration and optional --kernel-dist flag")
return parser
args = parser.parse_args()
def oMainLogger(args)
assert args.loglevel < 59
if coloredlogs is not None:
# https://pypi.org/project/coloredlogs/
@ -175,11 +175,14 @@ if __name__ == "__main__":
else:
logging.basicConfig(level=args.loglevel) # logging.INFO
logging.basicConfig(level=args.loglevel)
return LOG
if __name__ == "__main__":
parser = oMainParser()
args = parser.parse_args()
oMainLogger(args)
isos = gentooimgr.common.find_iso(args.download_dir)
if args.action == "run" and args.iso is None and len(isos) > 1:
LOG.error(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
sys.exit(1)
main(args)

1
src/ansible_gentooimgr/gentooimgr/install.py
View file

@ -314,6 +314,7 @@ def configure(args, config: dict):
os.makedirs(gentooimgr.config.GENTOO_MOUNT)
# disk prep
cfg = config
# must be root for linux-6.1.52-pentoo/certs/signing_key.pem
if not stepdone(1): step1_diskprep(args, cfg)
# mount root
if not stepdone(2): step2_mount(args, cfg)