update

src/ansible_gentooimgr/gentooimgr/#install.py#

@@ -0,0 +1,360 @@
+"""Configure a Gentoo guest with cloud image settings
+
+This step keeps track of how far it's gotten, so re-running this command
+will continue on if an error was to occur, unless --start-over flag is given.
+"""
+
+import os
+import sys
+import shutil
+import configparser
+from subprocess import Popen, PIPE
+import logging
+import traceback
+
+import gentooimgr.config
+import gentooimgr.configs
+import gentooimgr.common
+import gentooimgr.chroot
+import gentooimgr.kernel
+from gentooimgr import LOG
+from gentooimgr import HERE
+
+from gentooimgr.configs import *
+
+FILES_DIR = os.path.join(HERE, "..")
+
+def step1_diskprep(args, cfg):
+    LOG.info("\t:: Step 1: Disk Partitioning")
+    # http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
+    # http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
+    cmds = [
+            ['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
+            ['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
+            ['partprobe'],
+            ['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
+    ]
+    for c in cmds:
+        proc = Popen(c, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+
+    completestep(1, "diskprep")
+
+def step2_mount(args, cfg):
+    LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
+    proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
+    proc.communicate()
+    completestep(2, "mount")
+
+def step3_stage3(args, cfg):
+    LOG.info(f'\t:: Step 3: Stage3 Tarball')
+
+    stage3 = cfg.get("stage3") or args.stage3  # FIXME: auto detect stage3 images in mountpoint and add here
+    if not stage3:
+        stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
+
+
+    proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
+                  f'{cfg.get("mountpoint")}'])
+    proc.communicate()
+    completestep(3, "stage3")
+
+def step4_binds(args, cfg):
+    LOG.info(f'\t:: Step 4: Binding Filesystems')
+    gentooimgr.chroot.bind(verbose=False)
+    completestep(4, "binds")
+
+def step5_portage(args, cfg):
+    LOG.info(f'\t:: Step 5: Portage')
+    portage = cfg.get("portage") or args.portage
+    if not portage:
+        portage = gentooimgr.common.portage_from_dir(FILES_DIR)
+    proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
+    proc.communicate()
+    # Edit portage
+    portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
+    os.makedirs(portage_env, exist_ok=True)
+    with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
+        f.write('MAKEOPTS="-j1"\n')
+
+    env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
+    with open(env_path, 'w') as f:
+        f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
+
+    completestep(5, "portage")
+
+def step6_licenses(args, cfg):
+    LOG.info(f'\t:: Step 6: Licenses')
+    license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
+    os.makedirs(license_path, exist_ok=True)
+    for f, licenses in cfg.get("licensefiles", {}).items():
+        with open(os.path.join(license_path, f), 'w') as f:
+            f.write('\n'.join(licenses))
+    completestep(6, "license")
+
+def step7_repos(args, cfg):
+    LOG.info(f'\t:: Step 7: Repo Configuration')
+    repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
+    os.makedirs(repo_path, exist_ok=True)
+    # Copy from template
+    repo_file = os.path.join(repo_path, 'gentoo.conf')
+    shutil.copyfile(
+        os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
+        repo_file)
+    # Regex replace lines
+    cp = configparser.ConfigParser()
+    for repofile, data in cfg.get("repos", {}).items():
+        cp.read(cfg.get("mountpoint") + repofile)  # repofile should be absolute path, do not use os.path.join.
+        for section, d in data.items():
+            if section in cp:
+                for key, val in d.items():
+                    # Replace everything after the key with contents of value.
+                    # Sed is simpler than using regex for this purpose.
+                    cp.set(section, key, val)
+            else:
+                sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
+
+        cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
+
+    completestep(7, "repos")
+
+def step8_resolv(args, cfg):
+    LOG.info(f'\t:: Step 8: Resolv')
+    proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
+    proc.communicate()
+    # Copy all step files and python module to new chroot
+    os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
+    os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
+    completestep(8, "resolv")
+
+def step9_sync(args, cfg):
+    LOG.info(f"\t:: Step 9: sync")
+    LOG.info("\t\t:: Entering chroot")
+    os.chroot(cfg.get("mountpoint"))
+    os.chdir(os.sep)
+    os.system("source /etc/profile")
+    proc = Popen(["emerge", "--sync", "--quiet"])
+    proc.communicate()
+    LOG.info("\t\t:: Emerging base")
+    proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
+    proc.communicate()
+    completestep(9, "sync")
+
+def step10_emerge_pkgs(args, cfg):
+    LOG.info(f"\t:: Step 10: emerge pkgs")
+    packages = cfg.get("packages", {})
+    for oneshot_up in packages.get("oneshots", []):
+        proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
+        proc.communicate()
+
+    for single in packages.get("singles", []):
+        proc = Popen(["emerge", "-j1", single])
+        proc.communicate()
+
+    LOG.info(f"KERNEL PACKAGES {packages.get('kernel')}")
+    if packages.get("kernel", []):
+        cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
+        proc = Popen(cmd)
+        proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
+    cmd += packages.get("keepgoing", [])
+    proc = Popen(cmd)
+    proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads)]
+    cmd += packages.get("base", [])
+    cmd += packages.get("additional", [])
+    cmd += packages.get("bootloader", [])
+    LOG.info(cmd)
+    proc = Popen(cmd)
+    proc.communicate()
+    completestep(10, "pkgs")
+
+def step11_kernel(args, cfg):
+    # at this point, genkernel will be installed
+    LOG.info(f"\t:: Step 11: kernel")
+    proc = Popen(["eselect", "kernel", "set", "1"])
+    proc.communicate()
+    if not args.kernel_dist:
+        os.chdir(args.kernel_dir)
+        threads = str(gentooimgr.config.THREADS)
+        gentooimgr.kernel.build_kernel(args, cfg)
+
+    completestep(11, "kernel")
+
+def step12_grub(args, cfg):
+    LOG.info(f"\t:: Step 12: kernel")
+    proc = Popen(["grub-install", cfg.get('disk')])
+    proc.communicate()
+    code = proc.returncode
+    if code != 0:
+        sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
+        sys.exit(code)
+
+    with open("/etc/default/grub", 'w') as f:
+        f.write(f"{gentooimgr.kernel.GRUB_CFG}")
+
+    proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
+    proc.communicate()
+    completestep(12, "grub")
+
+def step13_serial(args, cfg):
+    LOG.info(f"\t:: Step 13: Serial")
+    os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
+    os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
+    completestep(13, "serial")
+
+def step14_services(args, cfg):
+    LOG.info(f"\t:: Step 14: Services")
+    for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
+                    "cloud-final", "ntpd", "nfsclient"]:
+        if args.profile == "systemd":
+            proc = Popen(["systemctl", "enable", service])
+        else:
+            proc = Popen(["rc-update", "add", service, "default"])
+        proc.communicate()
+
+    completestep(14, "services")
+
+def step15_ethnaming(args, cfg):
+    LOG.info(f"\t:: Step 15: Eth Naming")
+    completestep(15, "networking")
+
+def step16_sysconfig(args, cfg):
+    LOG.info(f"\t:: Step 16: Sysconfig")
+    with open("/etc/timezone", "w") as f:
+        f.write("UTC")
+    proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
+    proc.communicate()
+    with open("/etc/locale.gen", "a") as f:
+        f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
+    proc = Popen(["locale-gen"])
+    proc.communicate()
+    proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
+    proc.communicate()
+    proc = Popen(["env-update"])
+    proc.communicate()
+    with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
+        f.write("vm.swappiness = 0\n")
+
+    modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
+    os.makedirs(modloadpath, exist_ok=True)
+    with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
+        f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
+
+    cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
+    if not os.path.exists(cloudcfg):
+        os.makedirs(cloudcfg, exist_ok=True)
+        os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
+    with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
+        cfg.write(f"{CLOUD_YAML}")
+
+    os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
+
+    with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
+        tmpl.write(f"{HOST_TMPL}") # FIXME:
+
+    os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
+
+    proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
+    proc.communicate()
+
+    hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
+    with open(hostname, 'w') as f:
+        f.write(f"{HOSTNAME}\n")
+
+    os.chmod(hostname, 0o644)
+
+    proc = Popen(["eix-update"])
+    proc.communicate()
+
+    os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
+
+    completestep(16, "sysconfig")
+
+def step17_fstab(args, cfg):
+    LOG.info(f"\t:: Step 17: fstab")
+    with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
+        fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
+
+    completestep(17, "fstab")
+
+def completestep(step, stepname, prefix='/tmp'):
+    with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
+        f.write("done.")  # text in this file is not currently used.
+
+
+def getlaststep(prefix='/tmp'):
+    i = 1
+    found = False
+    while not found:
+        if os.path.exists(f"{i}.step"):
+            i += 1
+        else:
+            found = True
+
+    return i
+
+
+def stepdone(step, prefix='/tmp'):
+    return os.path.exists(os.path.join(prefix, f"{step}.step"))
+
+def configure(args, config: dict):
+    # Load configuration
+    if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
+        if not args.force:
+            # We aren't in a gentoo live cd are we?
+            sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
+                "If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
+            sys.exit(1)
+
+        else:
+            # Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
+            os.makedirs(gentooimgr.config.GENTOO_MOUNT)
+    # disk prep
+    cfg = config
+    # must be root for linux-6.1.52-pentoo/certs/signing_key.pem
+    if not stepdone(1): step1_diskprep(args, cfg)
+    # mount root
+    if not stepdone(2): step2_mount(args, cfg)
+    # extract stage
+    if not stepdone(3): step3_stage3(args, cfg)
+    # mount binds
+    if not stepdone(4): step4_binds(args, cfg)
+    # extract portage
+    if not stepdone(5): step5_portage(args, cfg)
+    # Set licenses
+    if not stepdone(6): step6_licenses(args, cfg)
+    # repos.conf
+    if not stepdone(7): step7_repos(args, cfg)
+    # portage env files and resolv.conf
+    if not stepdone(8): step8_resolv(args, cfg)
+    # emerge --sync
+    if not stepdone(9): step9_sync(args, cfg)
+    # bindist
+    if not stepdone(10): step10_emerge_pkgs(args, cfg)
+    # emerge packages
+    # configure & emerge kernel (use cloud configuration too)
+    if not stepdone(11): step11_kernel(args, cfg)
+    # grub
+    if not stepdone(12): step12_grub(args, cfg)
+    # enable serial console
+    if not stepdone(13): step13_serial(args, cfg)
+    # services
+    if not stepdone(14): step14_services(args, cfg)
+    # eth0 naming
+    # timezone
+    if not stepdone(15): step15_ethnaming(args, cfg)
+    # locale
+    # set some sysctl things
+    # set some dhcp things
+    # hostname
+    if not stepdone(16): step16_sysconfig(args, cfg)
+    # fstab
+    if not stepdone(17): step17_fstab(args, cfg)
+    # copy cloud cfg?
+    gentooimgr.chroot.unbind()
+    # Finish install processes like emaint and eix-update and news read
+
+

src/ansible_gentooimgr/gentooimgr/.#install.py

@@ -0,0 +1 @@
+root@pentoo.152064:1703733868

src/ansible_gentooimgr/gentooimgr/__main__.py

@@ -165,7 +165,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     assert args.loglevel < 59
-    if coloredlogs:
+    if coloredlogs is not None:
         # https://pypi.org/project/coloredlogs/
         coloredlogs.install(level=args.loglevel,
                         logger=LOG,

src/ansible_gentooimgr/gentooimgr/builder.py

@@ -8,7 +8,7 @@ import gentooimgr.qemu as qemu
 import gentooimgr.common
 import requests
 
-def build(args: argparse.Namespace, config: dict) -> None:
+def build(args: argparse.Namespace, config: dict) -> str:
     LOG.info(": build")
 
     iso = config.get("iso") or download.download(args)