added current gentooimgr

gentooimgr/__init__.py

@@ -0,0 +1,2 @@
+import os
+HERE = os.path.abspath(os.path.dirname(__file__))

gentooimgr/__main__.py

@@ -0,0 +1,154 @@
+import os
+import sys
+import json
+import argparse
+import pathlib
+import copy
+import gentooimgr.common
+import gentooimgr.config
+import gentooimgr.configs
+
+
+def main(args):
+    '''Gentoo Cloud Image Builder Utility'''
+    import gentooimgr.config
+    configjson = gentooimgr.config.determine_config(args)
+
+    if args.action == "build":
+        import gentooimgr.builder
+        gentooimgr.builder.build(args, configjson)
+
+    elif args.action == "run":
+        import gentooimgr.run
+        gentooimgr.run.run(args, configjson)
+
+    elif args.action == "test":
+        import gentooimgr.test
+
+    elif args.action == "clean":
+        import gentooimgr.clean
+
+    elif args.action == "status":
+        import gentooimgr.status
+        gentooimgr.status.print_template(args, configjson)
+
+    elif args.action == "install":
+        import gentooimgr.install
+        gentooimgr.install.configure(args, configjson)
+
+    elif args.action == "command":
+        import gentooimgr.command
+        gentooimgr.command.command(configjson)
+
+    elif args.action == "chroot":
+        import gentooimgr.chroot
+        gentooimgr.chroot.chroot(path=args.mountpoint, shell="/bin/bash")
+
+    elif args.action == "unchroot":
+        import gentooimgr.chroot
+        gentooimgr.chroot.unchroot(path=args.mountpoint)
+
+    elif args.action == "shrink":
+        import gentooimgr.shrink
+        fname = gentooimgr.shrink.shrink(args, configjson, stamp=args.stamp)
+        print(f"Shrunken image at {fname}, {os.path.getsize(fname)}")
+
+    elif args.action == "kernel":
+        import gentooimgr.kernel
+        gentooimgr.kernel.build_kernel(args, configjson)
+
+if __name__ == "__main__":
+    """Gentoo Cloud Image Builder Utility"""
+    parser = argparse.ArgumentParser(prog="gentooimgr", description="Gentoo Image Builder Utility")
+    parser.add_argument("-c", "--config", nargs='?', type=pathlib.Path,
+                        help="Path to a custom conf file")
+    parser.add_argument("--config-cloud", action="store_const", const="cloud.json", dest="config",
+                        help="Use cloud init configuration")
+    parser.add_argument("--config-base", action="store_const", const="base.json", dest="config",
+                        help="Use a minimal base Gentoo configuration")
+
+    parser.add_argument("-t", "--temporary-dir", nargs='?', type=pathlib.Path,
+                        default=os.getcwd(), help="Path to temporary directory for downloading files")
+    parser.add_argument("-j", "--threads", type=int, default=gentooimgr.config.THREADS,
+                        help="Number of threads to use for building and emerging software")
+    parser.add_argument("-d", "--download-dir", type=pathlib.Path, default=os.getcwd(),
+                        help="Path to the desired download directory (default: current)")
+    parser.add_argument("--openrc", dest="profile", action="store_const", const="openrc",
+                        help="Select OpenRC as the Gentoo Init System")
+    parser.add_argument("--systemd", dest="profile", action="store_const", const="systemd",
+                        help="Select SystemD as the Gentoo Init System")
+    parser.add_argument("-f", "--force", action="store_true",
+                        help="Let action occur at potential expense of data loss or errors (applies to clean and cloud-cfg)")
+    parser.add_argument("--format", default="qcow2", help="Image format to generate, default qcow2")
+    parser.add_argument("--portage", default=None, type=pathlib.Path, nargs='?',
+                            help="Extract the specified portage package onto the filesystem")
+    parser.add_argument("--stage3", default=None, type=pathlib.Path, nargs='?',
+                            help="Extract the specified stage3 package onto the filesystem")
+    parser.add_argument("--kernel-dir", default="/usr/src/linux",
+                               help="Where kernel is specified. By default uses the active linux kernel")
+    subparsers = parser.add_subparsers(help="gentooimgr actions", dest="action")
+    subparsers.required = True
+
+    # Build action
+    parser_build = subparsers.add_parser('build', help="Download and verify all the downloaded components for cloud image")
+    parser_build.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME, type=str, nargs='?',
+                              help="Specify the exact image (date) you want to build; ex: 20231112T170154Z. Defaults to downloading the latest image. If image exists, will automatically use that one instead of checking online.")
+    parser_build.add_argument("--size", default="12G", help="Size of image to build")
+    parser_build.add_argument("--no-verify", dest="verify", action="store_false", help="Do not verify downloaded iso")
+    parser_build.add_argument("--verify", dest="verify", action="store_true", default=True,
+                              help="Verify downloaded iso")
+    parser_build.add_argument("--redownload", action="store_true", help="Overwrite downloaded files")
+    parser_run = subparsers.add_parser('run', help="Run a Gentoo Image in QEMU to process it into a cloud image")
+
+    parser_run.add_argument("--iso", default=None, type=pathlib.Path, nargs='?',
+                            help="Mount the specified iso in qemu, should be reserved for live cd images")
+    parser_run.add_argument("image", default=gentooimgr.config.GENTOO_IMG_NAME,
+                            type=pathlib.Path, nargs="?",
+                            help="Run the specified image in qemu")
+    parser_run.add_argument("-m", "--mounts", nargs='+', default=[],
+                            help="Path to iso files to mount into the running qemu instance")
+
+    parser_test = subparsers.add_parser('test', help="Test whether image is a legitamite cloud configured image")
+
+    parser_clean = subparsers.add_parser('clean', help="Remove all downloaded files")
+    # --force also applies to clean action
+
+    parser_status = subparsers.add_parser('status', help="Review information, downloaded images and configurations")
+
+    parser_install = subparsers.add_parser("install", help="Install Gentoo on a qemu guest. Defaults to "
+                                           "--config-base with --kernel-dist if the respective --config or --kernel options are not provided.")
+    parser_install.add_argument("--kernel-dist", action="store_true",
+                              help="Use a distribution kernel in the installation. Overrides all other kernel options.")
+    parser_install.add_argument("--kernel-virtio", action="store_true", help="Include virtio support in non-dist kernels")
+    parser_install.add_argument("--kernel-g5", action="store_true", help="Include all kernel config options for PowerMac G5 compatibility")
+
+    parser_chroot = subparsers.add_parser("chroot", help="Bind mounts and enter chroot with shell on guest. Unmounts binds on shell exit")
+    parser_chroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
+                               help="Point to mount and run the chroot and shell")
+
+    parser_unchroot = subparsers.add_parser("unchroot", help="Unmounts chroot filesystems")
+    parser_unchroot.add_argument("mountpoint", nargs='?', default=gentooimgr.config.GENTOO_MOUNT,
+                               help="Point to mount and run the chroot and shell")
+
+    parser_cmd = subparsers.add_parser('command', help="Handle bind mounts and run command(s) in guest chroot, then unmount binds")
+    parser_cmd.add_argument("cmds", nargs='*',
+                            help="Commands to run (quote each command if more than one word, ie: \"grep 'foo'\" \"echo foo\")")
+
+    parser_shrink = subparsers.add_parser('shrink', help="Take a finalized Gentoo image and rearrange it for smaller size")
+    parser_shrink.add_argument("img", type=pathlib.Path, help="Image to shrink")
+    parser_shrink.add_argument("--stamp", nargs='?', default=None,
+                               help="By default a timestamp will be added to the image name, otherwise provide "
+                               "a hardcoded string to add to the image name. Result: gentoo-[stamp].img")
+
+    parser_kernel = subparsers.add_parser('kernel', help="Build the kernel based on configuration and optional --kernel-dist flag")
+
+
+    args = parser.parse_args()
+
+    isos = gentooimgr.common.find_iso(args.download_dir)
+    if args.action == "run" and args.iso is None and len(isos) > 1:
+        print(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
+        sys.exit(1)
+
+
+    main(args)

gentooimgr/builder.py

@@ -0,0 +1,21 @@
+import os
+import argparse
+import gentooimgr.config as config
+import gentooimgr.download as download
+import gentooimgr.qemu as qemu
+import gentooimgr.common
+import requests
+
+def build(args: argparse.Namespace, config: dict) -> None:
+
+    iso = config.get("iso") or download.download(args)
+    stage3 = config.get("stage3") or download.download_stage3(args)
+    portage = config.get("portage") or download.download_portage(args)
+    filename = f"{args.image}.{args.format}"
+    image = qemu.create_image(args, config)
+    if not os.path.exists(image):
+        raise Exception(f"Image {image} does not exist")
+
+    is_default = os.path.basename(image) == filename
+    print(image)
+    print(f"Image {image} build successfully.\nRun `python -m gentooimgr run{' ' + image if not is_default else ''} --iso {iso}`")

gentooimgr/chroot.py

@@ -0,0 +1,54 @@
+import os
+import sys
+from subprocess import Popen, PIPE
+import gentooimgr.config
+
+def bind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
+    mounts = [
+        ["mount", "--types", "proc", "/proc", os.path.join(mount, "proc")],
+        ["mount", "--rbind", "/sys", os.path.join(mount, "sys")],
+        ["mount", "--make-rslave", os.path.join(mount, "sys")],
+        ["mount", "--rbind", "/dev", os.path.join(mount, "dev")],
+        ["mount", "--make-rslave", os.path.join(mount, "dev")],
+        ["mount", "--bind", "/run", os.path.join(mount, "run")],
+        ["mount", "--make-slave", os.path.join(mount, "run")],
+    ]
+    for mcmd in mounts:
+        if verbose:
+            print(f"\t:: {' '.join(mcmd)}")
+        proc = Popen(mcmd, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            sys.exit(proc.returncode)
+
+def unbind(mount=gentooimgr.config.GENTOO_MOUNT, verbose=True):
+    os.chdir("/")
+    if not os.path.exists(mount):
+        sys.stderr.write(f"Mountpoint {mount} does not exist\n")
+        return
+
+    unmounts = [
+        ["umount", os.path.join(mount, 'dev', 'shm')],
+        ["umount", os.path.join(mount, 'dev', 'pts')],
+        ["umount", "-l", os.path.join(mount, 'dev')],
+        ["umount", "-R", mount]
+    ]
+    for uncmd in unmounts:
+        if verbose:
+            print(f"\t:: {' '.join(uncmd)}")
+        proc = Popen(uncmd)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            continue
+
+def chroot(path=gentooimgr.config.GENTOO_MOUNT, shell="/bin/bash"):
+    bind(mount=path)
+    os.chroot(path)
+    os.chdir(os.sep)
+    os.system(shell)
+    unchroot(path=path)  # May fail if we do this automatically
+
+def unchroot(path=gentooimgr.config.GENTOO_MOUNT):
+    unbind(mount=path)

gentooimgr/command.py

@@ -0,0 +1,12 @@
+import sys
+import gentooimgr.chroot
+
+def command(config, *args):
+    gentooimgr.chroot.bind()
+    for a in args:
+        proc = Popen(a, shell=True, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            sys.stderr.write(f"{stderr}\n")
+            break
+    gentooimgr.chroot.unbind()

gentooimgr/common.py

@@ -0,0 +1,119 @@
+import os
+import sys
+import time
+import copy
+import json
+from subprocess import Popen, PIPE
+
+import gentooimgr.config
+
+def older_than_a_day(fullpath):
+    if not os.path.exists(fullpath):
+        return True  # Don't fail on missing files
+    filetime = os.path.getmtime(fullpath)
+    return time.time() - filetime > gentooimgr.config.DAY_IN_SECONDS
+
+
+def find_iso(download_dir):
+    name = None
+    ext = None
+    found = []
+    for f in os.listdir(download_dir):
+        name, ext = os.path.splitext(f)
+        if ext == ".iso":
+            found.append(os.path.join(download_dir, f))
+
+    return found
+
+def make_iso_from_dir(mydir):
+    """ Generates an iso with gentooimgr inside it for use inside a live cd guest
+    :Returns:
+        path to iso that was created or NoneType if mydir is not found
+    """
+    if not os.path.exists(mydir):
+        return
+
+    print(f"\t:: Making ISO with dir of {mydir}")
+    path = os.path.join(mydir, "..", "cloudgen.iso")
+    proc = Popen(["mkisofs",
+        "--input-charset", "utf-8",
+        "-J",
+        "-r",
+        "-V", "gentooimgr",
+        "-m", "*.img",
+        "-m", "*.iso",
+        "-o", path,
+        mydir
+        ], stdout=PIPE, stderr=PIPE)
+    proc.communicate()
+
+    return path
+
+def portage_from_dir(d, filename=None):
+    """Find portage file from directory. Will do a check in os.listdir() for portage*.tar.bz2.
+
+    If a filename is provided, this function either returns that filename assuming it exists in d,
+    or return None. If filename is None, this looks through all entries for portage files and if
+    only one exists, returns it, otherwise None.
+    """
+    found = []
+    for f in os.listdir(d):
+        if filename is not None:
+            if filename == f:
+                found.append(f)
+        elif f.startswith("portage") and f.endswith(".tar.xz"):
+            found.append(f)
+
+    if len(found) > 1:
+        sys.stderr.write("\tEE: More than one portage file exists, please specify the exact portage file with --portage [file]  or remove all others\n")
+        sys.stderr.write(''.join([f"\t{f}\n" for f in found]))
+        sys.stderr.write(f"in {d}\n")
+        sys.exit(1)
+
+    return found[0] if found else None
+
+
+def stage3_from_dir(d, filename=None):
+    """Find stage3 file from directory. Will do a check in os.listdir() for stage3*.tar.xz.
+
+    If a filename is provided, this function either returns that filename assuming it exists in d,
+    or return None. If filename is None, this looks through all entries for stage3 files and if
+    only one exists, returns it, otherwise None.
+    """
+    found = []
+    for f in os.listdir(d):
+        if filename is not None:
+            if filename == f:
+                found.append(f)
+        elif f.startswith("stage3") and f.endswith(".tar.xz"):
+            found.append(f)
+
+    if len(found) > 1:
+        sys.stderr.write("More than one stage3 file exists, please specify the exact stage3 file or remove all others\n")
+        sys.stderr.write(''.join([f"\t{f}\n" for f in found]))
+        sys.stderr.write(f"in {d}\n")
+        return None
+
+    return found[0] if found else None
+
+
+def get_image_name(args, config):
+    image = config.get("imagename", "gentoo."+args.format)
+    if image is None:
+        image = "gentoo."+args.format
+    return image
+
+#
+# def load_config(args):
+#     cfg = generatecfg(args)
+#     if args.config:
+#         override = generatecfg(args, config=args.config)
+#         cfg.update(cfgoverride)
+#
+#     if cfg.get("portage") is None:
+#         cfg['portage'] = portage_from_dir(args.download_dir, filename=args.portage or cfg.get("portage"))
+#     if cfg.get("stage3") is None:
+#         cfg['stage3'] = stage3_from_dir(args.download_dir, filename=args.stage3 or cfg.get("stage3"))
+#
+#     return cfg
+#

gentooimgr/config.py

@@ -0,0 +1,118 @@
+import os
+import json
+import sys
+import argparse
+import gentooimgr.configs
+import multiprocessing
+
+# A day in seconds:
+DAY_IN_SECONDS = 60*60*24
+# Define threads to compile packages with
+THREADS = multiprocessing.cpu_count()
+# URL to latest image text file, defaults to amd64. This is parsed to find latest iso to download
+ARCHITECTURE = "amd64"
+GENTOO_BASE_ISO_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-install-{ARCHITECTURE}-minimal/"
+GENTOO_BASE_STAGE_OPENRC_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-openrc/"
+GENTOO_BASE_STAGE_SYSTEMD_URL = f"https://distfiles.gentoo.org/releases/{ARCHITECTURE}/autobuilds/current-stage3-{ARCHITECTURE}-systemd/"
+GENTOO_LATEST_ISO_FILE = f"latest-install-{ARCHITECTURE}-minimal.txt"
+GENTOO_LATEST_STAGE_OPENRC_FILE = f"latest-stage3-{ARCHITECTURE}-openrc.txt"
+GENTOO_LATEST_STAGE_SYSTEMD_FILE = f"latest-stage3-{ARCHITECTURE}-systemd.txt"
+GENTOO_PORTAGE_FILE = "http://distfiles.gentoo.org/snapshots/portage-latest.tar.xz"  # No architecture, no txt files to determine latest.
+
+GENTOO_MOUNT = "/mnt/gentoo"
+GENTOO_IMG_NAME = "gentoo.qcow2"
+
+GENTOO_FILE_HASH_RE = r"^Hash\: ([\w]*)$"
+GENTOO_FILE_ISO_RE = r"^(install-[\w\-_\.]*.iso) ([\d]*)"
+GENTOO_FILE_ISO_HASH_RE = r"^([\w]*)  (install-[\w\-_\.]*.iso)$"
+GENTOO_FILE_STAGE3_RE = r"^(stage3-[\w\-_\.]*.tar.*) ([\d]*)"
+GENTOO_FILE_STAGE3_HASH_RE = r"^([\w]*)  (stage3-[\w\-_\.]*.tar.*)$"
+# TODO: Repo regex to replace attributes, use function to do so as find key will change.
+
+def replace_repos_conf(key, value):
+    pass
+
+CLOUD_MODULES = [
+    "iscsi_tcp"
+]
+
+def load_config(path):
+    if os.path.exists(path):
+        with open(path, 'r') as f:
+            return json.loads(f.read())
+    return {}
+
+def load_default_config(config_name):
+    """This is called when a --config option is set. --kernel options update the resulting config, whether
+    it be 'base' or other.
+    If user is supplying their own configuration, this is not called.
+    """
+    name, ext = os.path.splitext(config_name)
+    if not name in gentooimgr.configs.KNOWN_CONFIGS:
+        return {}
+
+    with open(os.path.join(gentooimgr.configs.CONFIG_DIR, config_name), 'r') as f:
+        return json.loads(f.read())
+
+
+def inherit_config(config: dict) -> dict:
+    """Returns the json file that the inherit key specifies; will recursively update if inherit values are set.
+    """
+    configuration = load_default_config(config.get("inherit"))
+    if not configuration:
+        configuration = load_config(config.get("inherit"))
+
+    if not configuration:
+        sys.stderr.write(f"\tWW: Warning: Inherited configuration {config.get('inherit')} is not found.\n")
+        return {}
+
+    if configuration.get("inherit"):
+        configuration.update(inherit_config(configuration.get("inherit")))
+
+    return configuration
+
+def determine_config(args: argparse.Namespace) -> dict:
+    """Check argparser options and return the most valid configuration
+
+    The "package" key/value object overrides everything that is set, it does not update() them.
+    If you override "base" package set, it's exactly what you set. It makes more sense to do it this way.
+    For example, if you have a dist kernel config, you don't want the base.json to update and include all
+    non-dist kernel options as it would add a lot of used space for unused functionality.
+
+    The package set is only overridden in the top level json configuration file though;
+    If you have multiple inherits, those package sets will be combined before the parent package set overrides
+    with the keys that are set.
+
+    If you have base.json and base2.json that contain multiple layers of "base" packages, ie: base: ['foo'] and base2: ['bar']
+    then you will have in yours.json: packages { base: ['foo', 'bar'] } and unless you set "base", that is what you'll get.
+
+
+    If you check `status` action, it will flatten all configurations into one, so the "inherit" key will always be null.
+
+    :Returns:
+        - configuration from json to dict
+    """
+
+    # Check custom configuration
+    configuration = load_default_config(args.config or 'base.json')
+    if not configuration:
+        configuration = load_config(args.config)
+    if not configuration:
+        sys.stderr.write(f"\tWW: Warning: Configuration {args.config} is empty\n")
+    else:
+        if configuration.get("inherit"):
+            # newpkgs = configuration.get("packages", {})
+            inherited = inherit_config(configuration)
+            new_packages = configuration.get("packages", {})
+            old_packages = inherited.get("packages", {})
+            inherited.update(configuration)
+            # Set back old package dict and then update only what is set in new:
+            inherited['packages'] = old_packages
+            for key, pkgs in new_packages.items():
+                if pkgs:
+                    inherited['packages'][key] = pkgs
+
+            return inherited
+
+    return configuration
+

gentooimgr/configs/__init__.py

@@ -0,0 +1,153 @@
+import os
+
+CONFIG_DIR = os.path.abspath(os.path.dirname(__file__))
+
+__all__ = ["CONFIG_DIR", "CLOUD_YAML", "HOST_TMPL", "HOSTNAME", "KNOWN_CONFIGS"]
+
+# List of configurations that end in '.json' within the configs/ directory
+KNOWN_CONFIGS = [
+    "base",
+    "cloud"
+]
+
+# Currently we handle the writing of additional files by having data defined here and checking options.
+# this isn't ideal. TODO: Make this better.
+CLOUD_YAML = """
+# The top level settings are used as module
+# and system configuration.
+
+# A set of users which may be applied and/or used by various modules
+# when a 'default' entry is found it will reference the 'default_user'
+# from the distro configuration specified below
+users:
+   - default
+
+# If this is set, 'root' will not be able to ssh in and they
+# will get a message to login instead as the above $user (ubuntu)
+disable_root: true
+ssh_pwauth:   false
+
+# This will cause the set+update hostname module to not operate (if true)
+preserve_hostname: false
+
+# this may be helpful in certain scenarios
+# resize_rootfs_tmp: /dev
+
+syslog_fix_perms: root:root
+
+ssh_deletekeys: false
+ssh_genkeytypes: [rsa, dsa]
+
+# This can be 'template'
+# which would look for /etc/cloud/templates/hosts.gentoo.tmpl
+# or 'localhost'
+# or False / commented out to disable altogether
+manage_etc_hosts: template
+
+# Example datasource config
+# datasource:
+#    Ec2:
+#      metadata_urls: [ 'blah.com' ]
+#      timeout: 5 # (defaults to 50 seconds)
+#      max_wait: 10 # (defaults to 120 seconds)
+
+# The modules that run in the 'init' stage
+cloud_init_modules:
+ - seed_random
+ - bootcmd
+ - write-files
+ - growpart
+ - resizefs
+ - set_hostname
+ - update_hostname
+ - update_etc_hosts
+ - ca-certs
+ - users-groups
+ - ssh
+
+# The modules that run in the 'config' stage
+cloud_config_modules:
+# Emit the cloud config ready event
+# this can be used by upstart jobs for 'start on cloud-config'.
+ - disk_setup
+ - mounts
+ - ssh-import-id
+ - set-passwords
+ - package-update-upgrade-install
+ - timezone
+ - puppet
+ - chef
+ - salt-minion
+ - mcollective
+ - disable-ec2-metadata
+ - runcmd
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - scripts-vendor
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - ssh-authkey-fingerprints
+ - keys-to-console
+ - phone-home
+ - final-message
+ - power-state-change
+
+# System and/or distro specific settings
+# (not accessible to handlers/transforms)
+system_info:
+   # This will affect which distro class gets used
+   distro: gentoo
+   # Default user name + that default users groups (if added/used)
+   default_user:
+     name: gentoo
+     lock_passwd: True
+     gecos: Gentoo
+     groups: [users, wheel]
+     primary_group: users
+     no-user-group: true
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/bash
+   # Other config here will be given to the distro class and/or path classes
+   paths:
+      cloud_dir: /var/lib/cloud/
+      templates_dir: /etc/cloud/templates/
+"""
+
+HOST_TMPL = """
+## template:jinja
+{#
+This file /etc/cloud/templates/hosts.gentoo.tmpl is only utilized
+if enabled in cloud-config.  Specifically, in order to enable it
+you need to add the following to config:
+  manage_etc_hosts: template
+-#}
+# Your system has configured 'manage_etc_hosts' as 'template'.
+# As a result, if you wish for changes to this file to persist
+# then you will need to either
+# a.) make changes to the master file in /etc/cloud/templates/hosts.gentoo.tmpl
+# b.) change or remove the value of 'manage_etc_hosts' in
+#     /etc/cloud/cloud.cfg or cloud-config from user-data
+#
+# The following lines are desirable for IPv4 capable hosts
+127.0.0.1 {{fqdn}} {{hostname}}
+127.0.0.1 localhost.localdomain localhost
+127.0.0.1 localhost4.localdomain4 localhost4
+
+# The following lines are desirable for IPv6 capable hosts
+::1 {{fqdn}} {{hostname}}
+::1 localhost.localdomain localhost
+::1 localhost6.localdomain6 localhost6
+"""
+
+HOSTNAME = """
+# Set to the hostname of this machine
+if [ -f /etc/hostname ];then
+    hostname=$(cat /etc/hostname 2> /dev/null | cut -d"." -f1 2> /dev/null)
+else
+    hostname="localhost"
+fi
+"""
+

gentooimgr/configs/base.json

@@ -0,0 +1,81 @@
+{
+    "inherit": null,
+    "imgsize": "20G",
+    "memory": 4096,
+    "mountpoint": "/mnt/gentoo",
+    "imagename": null,
+    "initsys": "openrc",
+    "licensefiles": {
+        "kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
+    },
+    "kernel": {
+        "path": "/etc/kernels/config.d/gentooimgr-base.config"
+    },
+    "repos": {
+        "/etc/portage/repos.conf/gentoo.conf": {
+            "gentoo": {
+                "sync-uri": "rsync://192.168.254.20/gentoo-portage"
+            }
+        }
+    },
+    "packages": {
+        "base": [
+            "acpid",
+            "dmidecode",
+            "syslog-ng",
+            "cronie",
+            "dhcpcd",
+            "mlocate",
+            "xfsprogs",
+            "dosfstools",
+            "sudo",
+            "postfix",
+            "parted",
+            "portage-utils",
+            "bash-completion",
+            "gentoo-bashcomp",
+            "tmux",
+            "app-misc/screen",
+            "dev-vcs/git",
+            "net-misc/curl",
+            "usbutils",
+            "pciutils",
+            "logrotate",
+            "gptfdisk",
+            "sys-block/gpart",
+            "net-misc/ntp",
+            "net-fs/nfs-utils",
+            "linux-firmware"
+        ],
+        "additional": ["app-editors/vim"],
+        "oneshots": [
+            "portage"
+        ],
+        "singles": [
+            "app-portage/eix",
+            "dev-util/cmake"
+        ],
+        "keepgoing": [
+            "openssh"
+        ],
+        "bootloader": [
+            "grub:2"
+        ],
+        "kernel": [
+            "sys-kernel/genkernel",
+            "gentoo-sources",
+            "gentoolkit"
+        ]
+    },
+    "services": {
+        "syslog-ng": "default",
+        "cronie": "default",
+        "acpid": "default",
+        "ntp": "default"
+    },
+    "iso": null,
+    "portage": null,
+    "stage3": null,
+    "disk": "/dev/sda",
+    "partition": 1
+}

gentooimgr/configs/base.json.example

@@ -0,0 +1,77 @@
+{
+    "inherit": null,
+    "imgsize": "12G",
+    "memory": 4096,
+    "mountpoint": "/mnt/gentoo",
+    "imagename": null,
+    "initsys": "openrc",
+    "licensefiles": {
+        "kernel": ["sys-kernel/linux-firmware linux-fw-redistributable"]
+    },
+    "repos": {
+        "/etc/portage/repos.conf/gentoo.conf": {
+            "sync-uri": "rsync://192.168.254.20/gentoo-portage"
+        }
+    },
+    "packages": {
+        "base": [
+            "acpid",
+            "dmidecode",
+            "syslog-ng",
+            "cronie",
+            "dhcpcd",
+            "mlocate",
+            "xfsprogs",
+            "dosfstools",
+            "sudo",
+            "postfix",
+            "app-editors/vim",
+            "parted",
+            "portage-utils",
+            "bash-completion",
+            "gentoo-bashcomp",
+            "tmux",
+            "app-misc/screen",
+            "dev-vcs/git",
+            "net-misc/curl",
+            "usbutils",
+            "pciutils",
+            "logrotate",
+            "gptfdisk",
+            "sys-block/gpart",
+            "net-misc/ntp",
+            "net-fs/nfs-utils",
+            "linux-firmware"
+        ],
+        "additional": [],
+        "oneshots": [
+            "portage"
+        ],
+        "singles": [
+            "app-portage/eix",
+            "dev-util/cmake"
+        ],
+        "keepgoing": [
+            "openssh"
+        ],
+        "bootloader": [
+            "grub:2"
+        ],
+        "kernel": [
+            "sys-kernel/genkernel",
+            "gentoo-sources",
+            "gentoolkit"
+        ]
+    },
+    "services": {
+        "syslog-ng": "default",
+        "cronie": "default",
+        "acpid": "default",
+        "ntp": "default"
+    },
+    "iso": null,
+    "portage": null,
+    "stage3": null,
+    "disk": null,
+    "partition": 1
+}

gentooimgr/configs/cloud.json

@@ -0,0 +1,15 @@
+{
+    "inherit": "base.json",
+    "packages": {
+        "additional": [
+            "app-emulation/cloud-init",
+            "sys-block/open-iscsi"
+        ]
+    },
+    "disk": "/dev/vda",
+    "kernel": {
+        "path": "/etc/kernels/config.d/gentooimgr-cloud.config",
+        "config": "cloud.config"
+    },
+    "partition": 1
+}

gentooimgr/download.py

@@ -0,0 +1,191 @@
+"""Module to handle downloading and verification of Gentoo images
+
+To ensure accuracy, we re-download every .txt file if it's older than one day.
+We assume that people building a cloud configured image want what is most up to date.
+If you have a specific image you want built over and over regardless, create a config
+file and load it in using -c/--config that points GENTOO_* values to the files you want.
+"""
+
+import os
+import re
+import sys
+from datetime import date
+import hashlib
+import progressbar
+from urllib.request import urlretrieve
+import tempfile
+import gentooimgr.config as config
+from gentooimgr.common import older_than_a_day
+
+hashpattern =    re.compile(config.GENTOO_FILE_HASH_RE, re.MULTILINE)
+isopattern =     re.compile(config.GENTOO_FILE_ISO_RE, re.MULTILINE)
+isohashpattern = re.compile(config.GENTOO_FILE_ISO_HASH_RE, re.MULTILINE)
+stage3pattern  = re.compile(config.GENTOO_FILE_STAGE3_RE, re.MULTILINE)
+stage3hashpattern = re.compile(config.GENTOO_FILE_STAGE3_HASH_RE, re.MULTILINE)
+
+class DownloadProgressBar():
+    def __init__(self):
+        self.progress = None
+
+    def __call__(self, block_num, block_size, total_size):
+        if not self.progress:
+            self.progress = progressbar.ProgressBar(maxval=total_size)
+            self.progress.start()
+
+        downloaded = block_num * block_size
+        if downloaded < total_size:
+            self.progress.update(downloaded)
+        else:
+            self.progress.finish()
+
+def parse_latest_iso_text(fullpath) -> tuple:
+    """Returns a tuple of (hash type, iso name, iso bytes)"""
+    with open(fullpath) as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        m_iso = isopattern.search(content)
+        return (m_hash.group(1) if not m_hash is None else None,
+                m_iso.group(1) if not m_iso is None else None,
+                m_iso.group(2) if not m_iso is None else None,)
+
+def parse_latest_stage3_text(fullpath) -> tuple:
+    """Returns a tuple of (hash type, iso name, iso bytes)
+    """
+    with open(fullpath) as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        m_stage3 = stage3pattern.search(content)
+        return (m_hash.group(1) if not m_hash is None else None,
+                m_stage3.group(1) if not m_stage3 is None else None,
+                m_stage3.group(2) if not m_stage3 is None else None,)
+
+def verify(args, _type: str, baseurl: str, hashpattern, filename: str) -> bool:
+    """Downloads hash file and run a hash check on the file
+    :Parameters:
+        - args: Namespace of parsed arguments
+        - _type: str hash type
+        - baseurl: (remote) folder where hashsum file is contained
+        - hashpattern:
+        - filename: str name of file to check (used to download corresponding hash file)
+
+            A install-amd64-minimal-2023111iso2T170154Z.iso file will have a
+              install-amd64-minimal-20231112T170154Z.iso.sha256 for example.
+
+    :Returns:
+        Whether iso was verified using the specified hash
+
+    """
+    digest = hashlib.file_digest(open(os.path.join(args.download_dir, filename), 'rb'), _type.lower())
+    filename = filename+f".{_type.lower()}"  # Update to hash file
+    hashfile = os.path.join(baseurl, filename)
+    fullpath = os.path.join(args.download_dir, os.path.basename(hashfile))
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        print(f"Downloading {filename}")
+        urlretrieve(hashfile, fullpath, DownloadProgressBar())
+
+    hd = digest.hexdigest()
+    with open(fullpath, 'r') as f:
+        content = f.read()
+        m_hash = hashpattern.search(content)
+        _hash = m_hash.group(1)
+        assert hd == _hash, f"Hash mismatch {hd} != {_hash}"
+
+def download_stage3(args, url=None) -> str:
+    if url is None:
+        if args.profile == "systemd":
+            url = os.path.join(config.GENTOO_BASE_STAGE_SYSTEMD_URL, config.GENTOO_LATEST_STAGE_SYSTEMD_FILE)
+
+        else:
+            url = os.path.join(config.GENTOO_BASE_STAGE_OPENRC_URL, config.GENTOO_LATEST_STAGE_OPENRC_FILE)
+
+    filename = os.path.basename(url)
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        print(f"Downloading {filename}")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    hashtype, latest, size = parse_latest_stage3_text(fullpath)
+    size = int(size)
+
+    filename = latest
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload:
+        print(f"Downloading {filename}")
+        url = os.path.join(
+                config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
+                config.GENTOO_BASE_STAGE_OPENRC_URL,
+                filename)
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    # Verify byte size
+    stage3size = os.path.getsize(fullpath)
+    assert size == stage3size, f"Stage 3 size {size} does not match expected value {stage3size}."
+    verify(args, hashtype, config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
+           config.GENTOO_BASE_STAGE_OPENRC_URL,  stage3hashpattern, filename)
+    return fullpath
+
+
+def download_portage(args, url=None) -> str:
+    """Handle downloading of portage system for installation into cloud image
+
+    We always download the latest portage package and rename it to today's date.
+    If using today's date to grab portage, sometimes depending on timezone, the
+    package won't be available. If always using latest, worst case scenario is you
+    have a portage package a day late.
+
+
+    """
+    if url is None:
+        url = config.GENTOO_PORTAGE_FILE
+
+    base = os.path.basename(url)  # Uses 'latest' filename
+    today = date.today()
+    # Write latest to today's date so we don't constantly redownload, but
+    filename = base.replace("latest", "%d%d%d" % (today.year, today.month, today.day))
+    fullpath = os.path.join(args.download_dir, filename)
+    # Portage is always "latest" in this case, so definitely check if older than a day and redownload.
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        print(f"Downloading {filename} ({base})")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    return fullpath
+
+
+def download(args, url=None) -> str:
+    """Download txt file with iso name and hash type
+    :Parameters:
+        - args: Namespace with parsed arguments
+        - url: str or None. If None, will generate a url to the latest minimal install iso
+
+    :Returns:
+        Full path to the downloaded iso file
+
+    Will cause program to exit if iso byte size fails to match expected value.
+    """
+    if url is None:
+        url = os.path.join(config.GENTOO_BASE_ISO_URL, config.GENTOO_LATEST_ISO_FILE)
+
+    # Download the latest txt file
+    filename = os.path.basename(url)
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
+        print(f"Downloading {filename}")
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    hashtype, latest, size = parse_latest_iso_text(fullpath)
+    size = int(size)
+
+    # Download the iso file
+    filename = latest
+    fullpath = os.path.join(args.download_dir, filename)
+    if not os.path.exists(fullpath) or args.redownload :
+        print(f"Downloading {filename}")
+        url = os.path.join(config.GENTOO_BASE_ISO_URL, filename)
+        urlretrieve(url, fullpath, DownloadProgressBar())
+
+    # Verify byte size
+    isosize = os.path.getsize(fullpath)
+    assert size == isosize, f"ISO size {size} does not match expected value {isosize}."
+    verify(args, hashtype, config.GENTOO_BASE_ISO_URL, isohashpattern, filename)
+
+    return fullpath

gentooimgr/install.py

@@ -0,0 +1,355 @@
+"""Configure a Gentoo guest with cloud image settings
+
+This step keeps track of how far it's gotten, so re-running this command
+will continue on if an error was to occur, unless --start-over flag is given.
+"""
+
+import os
+import sys
+import shutil
+import configparser
+from subprocess import Popen, PIPE
+import gentooimgr.config
+import gentooimgr.configs
+import gentooimgr.common
+import gentooimgr.chroot
+import gentooimgr.kernel
+from gentooimgr import HERE
+
+from gentooimgr.configs import *
+
+FILES_DIR = os.path.join(HERE, "..")
+
+def step1_diskprep(args, cfg):
+    print("\t:: Step 1: Disk Partitioning")
+    # http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
+    # http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
+    cmds = [
+            ['parted', '-s', f'{cfg.get("disk")}', 'mklabel', 'msdos'],
+            ['parted', '-s', f'{cfg.get("disk")}', 'mkpart', 'primary', '2048s', '100%'],
+            ['partprobe'],
+            ['mkfs.ext4', '-FF', f'{cfg.get("disk")}{cfg.get("partition", 1)}']
+    ]
+    for c in cmds:
+        proc = Popen(c, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+
+    completestep(1, "diskprep")
+
+def step2_mount(args, cfg):
+    print(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
+    proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
+    proc.communicate()
+    completestep(2, "mount")
+
+def step3_stage3(args, cfg):
+    print(f'\t:: Step 3: Stage3 Tarball')
+
+    stage3 = cfg.get("stage3") or args.stage3  # FIXME: auto detect stage3 images in mountpoint and add here
+    if not stage3:
+        stage3 = gentooimgr.common.stage3_from_dir(FILES_DIR)
+
+
+    proc = Popen(["tar", "xpf", os.path.abspath(stage3), "--xattrs-include='*.*'", "--numeric-owner", "-C",
+                  f'{cfg.get("mountpoint")}'])
+    proc.communicate()
+    completestep(3, "stage3")
+
+def step4_binds(args, cfg):
+    print(f'\t:: Step 4: Binding Filesystems')
+    gentooimgr.chroot.bind(verbose=False)
+    completestep(4, "binds")
+
+def step5_portage(args, cfg):
+    print(f'\t:: Step 5: Portage')
+    portage = cfg.get("portage") or args.portage
+    if not portage:
+        portage = gentooimgr.common.portage_from_dir(FILES_DIR)
+    proc = Popen(["tar", "xpf", portage, "-C", f"{cfg.get('mountpoint')}/usr/"])
+    proc.communicate()
+    # Edit portage
+    portage_env = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'env')
+    os.makedirs(portage_env, exist_ok=True)
+    with open(os.path.join(portage_env, 'singlejob.conf'), 'w') as f:
+        f.write('MAKEOPTS="-j1"\n')
+
+    env_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.env')
+    with open(env_path, 'w') as f:
+        f.write("app-portage/eix singlejob.conf\ndev-util/maturin singlejob.conf\ndev-util/cmake singlejob.conf")
+
+    completestep(5, "portage")
+
+def step6_licenses(args, cfg):
+    print(f'\t:: Step 6: Licenses')
+    license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
+    os.makedirs(license_path, exist_ok=True)
+    for f, licenses in cfg.get("licensefiles", {}).items():
+        with open(os.path.join(license_path, f), 'w') as f:
+            f.write('\n'.join(licenses))
+    completestep(6, "license")
+
+def step7_repos(args, cfg):
+    print(f'\t:: Step 7: Repo Configuration')
+    repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
+    os.makedirs(repo_path, exist_ok=True)
+    # Copy from template
+    repo_file = os.path.join(repo_path, 'gentoo.conf')
+    shutil.copyfile(
+        os.path.join(cfg.get("mountpoint"), 'usr', 'share', 'portage', 'config', 'repos.conf'),
+        repo_file)
+    # Regex replace lines
+    cp = configparser.ConfigParser()
+    for repofile, data in cfg.get("repos", {}).items():
+        cp.read(cfg.get("mountpoint") + repofile)  # repofile should be absolute path, do not use os.path.join.
+        for section, d in data.items():
+            if section in cp:
+                for key, val in d.items():
+                    # Replace everything after the key with contents of value.
+                    # Sed is simpler than using regex for this purpose.
+                    cp.set(section, key, val)
+            else:
+                sys.stderr.write(f"\tWW No section {section} in {repofile}\n")
+
+        cp.write(open(cfg.get("mountpoint") + repofile, 'w'))
+
+    completestep(7, "repos")
+
+def step8_resolv(args, cfg):
+    print(f'\t:: Step 8: Resolv')
+    proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
+    proc.communicate()
+    # Copy all step files and python module to new chroot
+    os.system(f"cp /tmp/*.step {cfg.get('mountpoint')}/tmp")
+    os.system(f"cp -r . {cfg.get('mountpoint')}/mnt/")
+    completestep(8, "resolv")
+
+def step9_sync(args, cfg):
+    print(f"\t:: Step 9: sync")
+    print("\t\t:: Entering chroot")
+    os.chroot(cfg.get("mountpoint"))
+    os.chdir(os.sep)
+    os.system("source /etc/profile")
+    proc = Popen(["emerge", "--sync", "--quiet"])
+    proc.communicate()
+    print("\t\t:: Emerging base")
+    proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
+    proc.communicate()
+    completestep(9, "sync")
+
+def step10_emerge_pkgs(args, cfg):
+    print(f"\t:: Step 10: emerge pkgs")
+    packages = cfg.get("packages", {})
+    for oneshot_up in packages.get("oneshots", []):
+        proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
+        proc.communicate()
+
+    for single in packages.get("singles", []):
+        proc = Popen(["emerge", "-j1", single])
+        proc.communicate()
+
+    print("KERNEL PACKAGES", packages.get("kernel"))
+    if packages.get("kernel", []):
+        cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
+        proc = Popen(cmd)
+        proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads), "--keep-going"]
+    cmd += packages.get("keepgoing", [])
+    proc = Popen(cmd)
+    proc.communicate()
+
+    cmd = ["emerge", "-j", str(args.threads)]
+    cmd += packages.get("base", [])
+    cmd += packages.get("additional", [])
+    cmd += packages.get("bootloader", [])
+    print(cmd)
+    proc = Popen(cmd)
+    proc.communicate()
+    completestep(10, "pkgs")
+
+def step11_kernel(args, cfg):
+    # at this point, genkernel will be installed
+    print(f"\t:: Step 11: kernel")
+    proc = Popen(["eselect", "kernel", "set", "1"])
+    proc.communicate()
+    if not args.kernel_dist:
+        os.chdir(args.kernel_dir)
+        threads = str(gentooimgr.config.THREADS)
+        gentooimgr.kernel.build_kernel(args, cfg)
+
+    completestep(11, "kernel")
+
+def step12_grub(args, cfg):
+    print(f"\t:: Step 12: kernel")
+    proc = Popen(["grub-install", cfg.get('disk')])
+    proc.communicate()
+    code = proc.returncode
+    if code != 0:
+        sys.stderr.write(f"Failed to install grub on {cfg.get('disk')}\n")
+        sys.exit(code)
+
+    with open("/etc/default/grub", 'w') as f:
+        f.write(f"{gentooimgr.kernel.GRUB_CFG}")
+
+    proc = Popen(["grub-mkconfig", "-o", "/boot/grub/grub.cfg"])
+    proc.communicate()
+    completestep(12, "grub")
+
+def step13_serial(args, cfg):
+    print(f"\t:: Step 13: Serial")
+    os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
+    os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
+    completestep(13, "serial")
+
+def step14_services(args, cfg):
+    print(f"\t:: Step 14: Services")
+    for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
+                    "cloud-final", "ntpd", "nfsclient"]:
+        if args.profile == "systemd":
+            proc = Popen(["systemctl", "enable", service])
+        else:
+            proc = Popen(["rc-update", "add", service, "default"])
+        proc.communicate()
+
+    completestep(14, "services")
+
+def step15_ethnaming(args, cfg):
+    print(f"\t:: Step 15: Eth Naming")
+    completestep(15, "networking")
+
+def step16_sysconfig(args, cfg):
+    print(f"\t:: Step 16: Sysconfig")
+    with open("/etc/timezone", "w") as f:
+        f.write("UTC")
+    proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
+    proc.communicate()
+    with open("/etc/locale.gen", "a") as f:
+        f.write("en_US.UTF-8 UTF-8\nen_US ISO-8859-1\n")
+    proc = Popen(["locale-gen"])
+    proc.communicate()
+    proc = Popen(["eselect", "locale", "set", "en_US.utf8"])
+    proc.communicate()
+    proc = Popen(["env-update"])
+    proc.communicate()
+    with open('/etc/sysctl.d/swappiness.conf', 'w') as f:
+        f.write("vm.swappiness = 0\n")
+
+    modloadpath = os.path.join(os.sep, 'etc', 'modules-load.d')
+    os.makedirs(modloadpath, exist_ok=True)
+    with open(os.path.join(modloadpath, 'cloud-modules.conf'), 'w') as f:
+        f.write('\n'.join(gentooimgr.config.CLOUD_MODULES))
+
+    cloudcfg = os.path.join(os.sep, 'etc', 'cloud')
+    if not os.path.exists(cloudcfg):
+        os.makedirs(cloudcfg, exist_ok=True)
+        os.makedirs(os.path.join(cloudcfg, 'templates'), exist_ok=True)
+    with open(os.path.join(cloudcfg, 'cloud.cfg'), 'w') as cfg:
+        cfg.write(f"{CLOUD_YAML}")
+
+    os.chmod(os.path.join(cloudcfg, "cloud.cfg"), 0o644)
+
+    with open(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 'w') as tmpl:
+        tmpl.write(f"{HOST_TMPL}") # FIXME:
+
+    os.chmod(os.path.join(cloudcfg, "templates", "hosts.gentoo.tmpl"), 0o644)
+
+    proc = Popen("sed -i 's/domain_name\,\ domain_search\,\ host_name/domain_search/g' /etc/dhcpcd.conf", shell=True)
+    proc.communicate()
+
+    hostname = os.path.join(os.sep, 'etc', 'conf.d', 'hostname')
+    with open(hostname, 'w') as f:
+        f.write(f"{HOSTNAME}\n")
+
+    os.chmod(hostname, 0o644)
+
+    proc = Popen(["eix-update"])
+    proc.communicate()
+
+    os.remove(os.path.join(os.sep, 'etc', 'resolv.conf'))
+
+    completestep(16, "sysconfig")
+
+def step17_fstab(args, cfg):
+    print(f"\t:: Step 17: fstab")
+    with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
+        fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
+
+    completestep(17, "fstab")
+
+def completestep(step, stepname, prefix='/tmp'):
+    with open(os.path.join(prefix, f"{step}.step"), 'w') as f:
+        f.write("done.")  # text in this file is not currently used.
+
+
+def getlaststep(prefix='/tmp'):
+    i = 1
+    found = False
+    while not found:
+        if os.path.exists(f"{i}.step"):
+            i += 1
+        else:
+            found = True
+
+    return i
+
+
+def stepdone(step, prefix='/tmp'):
+    return os.path.exists(os.path.join(prefix, f"{step}.step"))
+
+def configure(args, config: dict):
+    # Load configuration
+    if not os.path.exists(gentooimgr.config.GENTOO_MOUNT):
+        if not args.force:
+            # We aren't in a gentoo live cd are we?
+            sys.stderr.write("Your system doesn't look like a gentoo live cd, exiting for safety.\n"
+                "If you want to continue, use --force option and re-run `python -m gentooimgr install` with your configuration\n")
+            sys.exit(1)
+
+        else:
+            # Assume we are root as per live cd, otherwise user should run this as root as a secondary confirmation
+            os.makedirs(gentooimgr.config.GENTOO_MOUNT)
+    # disk prep
+    cfg = config
+    if not stepdone(1): step1_diskprep(args, cfg)
+    # mount root
+    if not stepdone(2): step2_mount(args, cfg)
+    # extract stage
+    if not stepdone(3): step3_stage3(args, cfg)
+    # mount binds
+    if not stepdone(4): step4_binds(args, cfg)
+    # extract portage
+    if not stepdone(5): step5_portage(args, cfg)
+    # Set licenses
+    if not stepdone(6): step6_licenses(args, cfg)
+    # repos.conf
+    if not stepdone(7): step7_repos(args, cfg)
+    # portage env files and resolv.conf
+    if not stepdone(8): step8_resolv(args, cfg)
+    # emerge --sync
+    if not stepdone(9): step9_sync(args, cfg)
+    # bindist
+    if not stepdone(10): step10_emerge_pkgs(args, cfg)
+    # emerge packages
+    # configure & emerge kernel (use cloud configuration too)
+    if not stepdone(11): step11_kernel(args, cfg)
+    # grub
+    if not stepdone(12): step12_grub(args, cfg)
+    # enable serial console
+    if not stepdone(13): step13_serial(args, cfg)
+    # services
+    if not stepdone(14): step14_services(args, cfg)
+    # eth0 naming
+    # timezone
+    if not stepdone(15): step15_ethnaming(args, cfg)
+    # locale
+    # set some sysctl things
+    # set some dhcp things
+    # hostname
+    if not stepdone(16): step16_sysconfig(args, cfg)
+    # fstab
+    if not stepdone(17): step17_fstab(args, cfg)
+    # copy cloud cfg?
+    gentooimgr.chroot.unbind()
+    # Finish install processes like emaint and eix-update and news read
+
+

gentooimgr/kernel.py

@@ -0,0 +1,128 @@
+import os
+import sys
+import re
+import shutil
+import datetime
+from subprocess import Popen, PIPE
+import time
+
+import gentooimgr.configs
+
+DEFAULT_KERNEL_CONFIG_PATH = os.path.join(os.sep, 'etc', 'kernel', 'default.config')
+
+def kernel_conf_apply(args, config):
+    """Kernel configuration is a direct copy of a full complete config file"""
+    fname, ext = os.path.splitext(args.config)
+    # Default is the json file's name but with .config extension.
+    kernelconfig = os.path.join(gentooimgr.configs.CONFIG_DIR,
+                                config.get("kernel", {}).get("config", f"{fname}.config"))
+    kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
+    if os.path.exists(kernelpath):
+        os.remove(kernelpath)
+    else:
+        # Ensure if we have directories specified that they exist
+        os.makedirs(os.path.dirname(kernelpath), exist_ok=True)
+
+    shutil.copyfile(kernelconfig, kernelpath)
+
+def build_kernel(args, config):
+    if config.get("kernel", {}).get("config") is None:
+        kernel_default_config(args, config)
+    os.chdir(args.kernel_dir)
+    kernelpath = config.get("kernel", {}).get("path", DEFAULT_KERNEL_CONFIG_PATH)
+    kernel_conf_apply(args, config)
+    proc = Popen(['genkernel', f'--kernel-config={kernelpath}', '--save-config', '--no-menuconfig', 'all'])
+    proc.communicate()
+    kernel_save_config(args, config)
+
+def kernel_default_config(args, config):
+    os.chdir(args.kernel_dir)
+    proc = Popen(["make", "defconfig"])
+    proc.communicate()
+
+def kernel_save_config(args, config):
+    os.chdir(args.kernel_dir)
+    """Saves the current .config file"""
+    proc = Popen(["make", "savedefconfig"])
+    proc.communicate()
+
+GRUB_CFG = """
+# Copyright 1999-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-x86/sys-boot/grub/files/grub.default-3,v 1.5 2015/03/25 01:58:00 floppym Exp $
+#
+# To populate all changes in this file you need to regenerate your
+# grub configuration file afterwards:
+#     'grub2-mkconfig -o /boot/grub/grub.cfg'
+#
+# See the grub info page for documentation on possible variables and
+# their associated values.
+
+GRUB_DISTRIBUTOR="Gentoo"
+
+# Default menu entry
+#GRUB_DEFAULT=0
+
+# Boot the default entry this many seconds after the menu is displayed
+#GRUB_TIMEOUT=5
+#GRUB_TIMEOUT_STYLE=menu
+
+# Append parameters to the linux kernel command line
+# openrc only spits to the last console=tty
+GRUB_CMDLINE_LINUX="net.ifnames=0 vga=791 console=tty0 console=ttyS0,115200"
+#
+# Examples:
+#
+# Boot with network interface renaming disabled
+# GRUB_CMDLINE_LINUX="net.ifnames=0"
+#
+# Boot with systemd instead of sysvinit (openrc)
+# GRUB_CMDLINE_LINUX="init=/usr/lib/systemd/systemd"
+
+# Append parameters to the linux kernel command line for non-recovery entries
+#GRUB_CMDLINE_LINUX_DEFAULT=""
+
+# Uncomment to disable graphical terminal (grub-pc only)
+GRUB_TERMINAL="serial console"
+GRUB_SERIAL_COMMAND="serial --speed=115200"
+#GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+
+# The resolution used on graphical terminal.
+# Note that you can use only modes which your graphic card supports via VBE.
+# You can see them in real GRUB with the command `vbeinfo'.
+#GRUB_GFXMODE=640x480
+
+# Set to 'text' to force the Linux kernel to boot in normal text
+# mode, 'keep' to preserve the graphics mode set using
+# 'GRUB_GFXMODE', 'WIDTHxHEIGHT'['xDEPTH'] to set a particular
+# graphics mode, or a sequence of these separated by commas or
+# semicolons to try several modes in sequence.
+#GRUB_GFXPAYLOAD_LINUX=
+
+# Path to theme spec txt file.
+# The starfield is by default provided with use truetype.
+# NOTE: when enabling custom theme, ensure you have required font/etc.
+#GRUB_THEME="/boot/grub/themes/starfield/theme.txt"
+
+# Background image used on graphical terminal.
+# Can be in various bitmap formats.
+#GRUB_BACKGROUND="/boot/grub/mybackground.png"
+
+# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to kernel
+#GRUB_DISABLE_LINUX_UUID=true
+
+# Uncomment to disable generation of recovery mode menu entries
+#GRUB_DISABLE_RECOVERY=true
+
+# Uncomment to disable generation of the submenu and put all choices on
+# the top-level menu.
+# Besides the visual affect of no sub menu, this makes navigation of the
+# menu easier for a user who can't see the screen.
+GRUB_DISABLE_SUBMENU=y
+
+# Uncomment to play a tone when the main menu is displayed.
+# This is useful, for example, to allow users who can't see the screen
+# to know when they can make a choice on the menu.
+#GRUB_INIT_TUNE="60 800 1"
+
+"""

gentooimgr/qemu.py

@@ -0,0 +1,84 @@
+"""Qemu commands to run and handle the image"""
+import os
+import sys
+import argparse
+from subprocess import Popen, PIPE
+import gentooimgr.config
+import gentooimgr.common
+def create_image(args, config: dict, overwrite: bool = False) -> str:
+    """Creates an image (.img) file using qemu that will be used to create the cloud image
+
+    :Parameters:
+        - config: dictionary/json configuration containing required information
+        - overwrite: if True, run_image() will call this and re-create.
+
+
+    :Returns:
+        Full path to image file produced by qemu
+    """
+
+    image = gentooimgr.common.get_image_name(args, config)
+    name, ext = os.path.splitext(image)
+    if os.path.exists(image) and not overwrite:
+        return os.path.abspath(image)
+
+    cmd = ['qemu-img', 'create', '-f', ext[1:], image, str(config.get("imgsize", "12G"))]
+    proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    stdout, stderr = proc.communicate()
+    return os.path.abspath(image)
+
+def run_image(
+    args: argparse.Namespace,
+    config: dict,
+    mounts=[]):
+    """Handle mounts and run the live cd image
+
+        - mount_isos: list of iso paths to mount in qemu as disks.
+    """
+    iso = config.get("iso")
+    if iso is None:
+        iso = gentooimgr.common.find_iso(
+            os.path.join(
+                os.path.abspath(os.path.dirname(__file__)),
+                ".."
+            )
+        )
+
+    if isinstance(iso, list):
+        iso = iso[0]
+
+
+    image = gentooimgr.common.get_image_name(args, config)
+    qmounts = []
+    mounts.extend(args.mounts)
+    for i in mounts:
+        qmounts.append("-drive")
+        qmounts.append(f"file={i},media=cdrom")
+
+    threads = args.threads
+    cmd = [
+        "qemu-system-x86_64",
+        "-enable-kvm",
+        "-boot", "d",
+        "-m", str(config.get("memory", 2048)),
+        "-smp", str(threads),
+        "-drive", f"file={image},if=virtio,index=0",
+        "-cdrom", iso,
+        "-net", "nic,model=virtio",
+        "-net", "user",
+        "-vga", "virtio",
+        "-cpu", "kvm64",
+        "-chardev", "file,id=charserial0,path=gentoo.log",
+        "-device", "isa-serial,chardev=charserial0,id=serial0",
+        "-chardev", "pty,id=charserial1",
+        "-device", "isa-serial,chardev=charserial1,id=serial1"
+    ]
+    cmd += qmounts
+    print(cmd)
+    proc = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    stdout, stderr = proc.communicate()
+    if stderr:
+        sys.stderr.write(str(stderr))
+        sys.stderr.write("\n")
+
+

gentooimgr/run.py

@@ -0,0 +1,28 @@
+import os
+import gentooimgr.config
+import gentooimgr.qemu
+import gentooimgr.common
+
+def run(args, config: dict):
+    mounts = args.mounts
+    # Specified image or look for gentoo.{img,qcow2}
+    image = config.get("imagename") or args.image or gentooimgr.qemu.create_image()
+    # We need to package up our gentooimgr package into an iso and mount it to the running image
+    # Why? basic gentoo livecd has no git and no pip installer. We want install to be simple
+    # and use the same common codebase.
+
+    # This will require a couple mount commands to function though.
+    main_iso = gentooimgr.common.make_iso_from_dir(os.path.join(
+        os.path.abspath(os.path.dirname(__file__)),
+        ".."
+    ))
+
+    print(args)
+    print(main_iso)
+    gentooimgr.qemu.run_image(
+        args,
+        config,
+        # Add our generated mount and livecd (assumed)
+        mounts=[main_iso]
+    )
+    print("done")

gentooimgr/shrink.py

@@ -0,0 +1,15 @@
+import os
+import datetime
+from subprocess import PIPE, Popen
+
+def shrink(args, config,  stamp=None):
+    if stamp is None:
+        dt = datetime.datetime.utcnow()
+        # 0 padded month and day timestamp
+        stamp = f"{dt.year}-{dt.month:02d}-{dt.day:02d}"
+    name, ext = os.path.splitext(config.get("imagename") or args.img)
+    # ext includes the .
+    filename = f"{name}-{stamp}{ext}"
+    proc = Popen(["virt-sparsify", "--compress", args.img, filename])
+    proc.communicate()
+    return filename

gentooimgr/status.py

@@ -0,0 +1,28 @@
+import json
+import gentooimgr.config
+import gentooimgr.configs
+
+def print_template(args, configjson):
+    print(f"""------------------------ STATUS ------------------------
+
+CPU_THREADS = {args.threads or 1}
+TEMPORARY_DIRECTORY = {args.temporary_dir}
+PROFILE = {args.profile}
+""")
+    print(f"CONFIG {args.config}")
+    print(json.dumps(configjson, sort_keys=True, indent=4))
+
+    # inherit = configjson.get("inherit")
+    # if inherit:
+    #     print(f"CONFIG {inherit}")
+    #     j = gentooimgr.config.load_default_config(inherit)
+    #     if not j:
+    #         j = gentooimgr.config.load_config(inherit)
+    #
+    #     print(json.dumps(j, sort_keys=True, indent=4))
+
+    # print(f"""------------------------ PACKAGES ------------------------""")
+    # for k, v in configjson.get("packages").items():
+    #     print(k.upper())
+    #     print("\t" + '\n\t'.join(v))
+    #     print()